## libraries
library(rgl)
library(TCGAbiolinks)
#library(TCGA2STAT)
library(caret)
## Loading required package: lattice
## Loading required package: ggplot2
library(ggplot2)
library(MASS)
library(heatmap.plus)
library(reshape2)
library(RColorBrewer)
library(ConsensusClusterPlus)
library(sigclust)
library(pheatmap)
library(tsne)
library(gplots)
## 
## Attaching package: 'gplots'
## The following object is masked from 'package:stats':
## 
##     lowess
library(ggradar)
library(doMC)
## Loading required package: foreach
## Loading required package: iterators
## Loading required package: parallel
library(dplyr)
## 
## Attaching package: 'dplyr'
## The following object is masked from 'package:MASS':
## 
##     select
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
library(tidyverse)
## ── Attaching packages ─────────────────────────────────────── tidyverse 1.3.0 ──
## ✓ tibble  3.0.4     ✓ purrr   0.3.4
## ✓ tidyr   1.1.2     ✓ stringr 1.4.0
## ✓ readr   1.4.0     ✓ forcats 0.5.0
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## x purrr::accumulate() masks foreach::accumulate()
## x dplyr::filter()     masks stats::filter()
## x dplyr::lag()        masks stats::lag()
## x purrr::lift()       masks caret::lift()
## x dplyr::select()     masks MASS::select()
## x purrr::when()       masks foreach::when()
library(plyr)
## ------------------------------------------------------------------------------
## You have loaded plyr after dplyr - this is likely to cause problems.
## If you need functions from both plyr and dplyr, please load plyr first, then dplyr:
## library(plyr); library(dplyr)
## ------------------------------------------------------------------------------
## 
## Attaching package: 'plyr'
## The following object is masked from 'package:purrr':
## 
##     compact
## The following objects are masked from 'package:dplyr':
## 
##     arrange, count, desc, failwith, id, mutate, rename, summarise,
##     summarize
library(MLmetrics)
## 
## Attaching package: 'MLmetrics'
## The following objects are masked from 'package:caret':
## 
##     MAE, RMSE
## The following object is masked from 'package:base':
## 
##     Recall
library(sparcl)
# --- importing data  ---#
expression_data_dir <- '/Users/mohama32/Documents/projects/GTCancerClassifier/OGFGT_data/'
meta_data_dir <- '/Users/mohama32/Documents/projects/GTCancerClassifier/OGFGT/data'

# import expression data
gt_types_file <- file.path(expression_data_dir, 'Glycosyltransferase_TCGA_RNASeq2_RSEM.txt')
gt_all = read_delim(gt_types_file, col_names = T, delim=" ")
## 
## ── Column specification ────────────────────────────────────────────────────────
## cols(
##   .default = col_double(),
##   .id = col_character()
## )
## ℹ Use `spec()` for the full column specifications.
# import list of GT genes
gt_file <- file.path(meta_data_dir, 'GT_Genes2.txt')
gt <- read_tsv(gt_file, col_names = FALSE)
## 
## ── Column specification ────────────────────────────────────────────────────────
## cols(
##   X1 = col_character()
## )
set.seed(1234)
intrain     = createDataPartition(gt_all$.id, p=.7, list=FALSE)
training    = gt_all[intrain,]
testing     = gt_all[-intrain,]
# sample selection
# we may use PCA to better show that certain types can be collapsed in an unsupervised way. I could not do that since the number of types is too many**
lda.fit = train(.id ~ ., data = training, method = "lda")
predictions = predict(lda.fit, testing[,-1])
confusionMatrix(predictions, as.factor(testing$.id))
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction ACC BLCA BRCA CESC CHOL COAD COADREAD DLBC ESCA GBM GBMLGG HNSC KICH
##   ACC       20    0    0    0    0    0        0    0    0   0      0    0    0
##   BLCA       0   81    4    5    1    0        1    0    0   0      0    0    0
##   BRCA       0    0  265    1    0    0        0    0    0   0      0    1    0
##   CESC       0   17    3   43    0    0        1    0    2   0      0   12    0
##   CHOL       0    0    0    0    3    0        0    0    0   0      0    0    0
##   COAD       0    0    0    0    0   19       35    0    1   0      0    1    0
##   COADREAD   0    0    0    0    0   46       57    0    0   0      0    0    0
##   DLBC       1    0    4    2    0    0        0    9    0   0      0    2    0
##   ESCA       0    0    1    0    0    0        0    0    8   0      0    0    0
##   GBM        0    0    0    0    0    0        0    0    0  40     66    0    0
##   GBMLGG     0    0    0    0    0    0        0    0    0   2     38    0    0
##   HNSC       0   13    4   21    0    0        0    0   13   0      0  117    0
##   KICH       0    0    0    0    0    0        0    0    0   0      0    0   16
##   KIPAN      0    0    0    0    0    0        0    0    0   0      0    0    0
##   KIRC       0    0    0    0    0    0        0    0    0   0      0    0    0
##   KIRP       0    0    0    0    0    0        0    0    0   0      0    0    0
##   LGG        0    0    0    0    0    0        0    0    0   0     95    0    0
##   LIHC       0    0    0    1    0    0        0    1    0   0      0    0    0
##   LUAD       0    0    2    0    2    0        1    0    0   0      0    0    0
##   LUSC       0    6    9    5    0    0        0    0   10   0      0   20    0
##   MESO       0    0    0    0    0    0        0    0    0   0      0    1    0
##   OV         0    0    1    2    1    0        0    0    1   0      0    0    0
##   PAAD       0    0    0    1    2    6        3    0    0   0      0    0    0
##   PCPG       0    0    1    0    0    0        0    0    0   0      0    0    0
##   PRAD       0    0    9    0    0    0        0    0    0   0      0    0    0
##   READ       0    0    0    1    0    9       10    0    1   0      0    0    0
##   SARC       1    1    1    0    0    1        0    0    1   2      1    1    0
##   SKCM       0    0    2    0    0    0        0    0    0   0      0    0    1
##   STAD       0    0    0    0    0    2        5    0   18   0      0    0    0
##   TGCT       0    0    0    0    0    0        0    2    0   0      0    0    0
##   THCA       0    0    2    0    0    0        0    0    0   0      0    0    0
##   THYM       1    1    0    2    1    0        0    1    0   0      0    0    0
##   UCEC       0    1   15    7    0    0        0    0    0   0      0    0    0
##   UCS        0    2    3    0    0    1        0    1    0   1      0    1    1
##   UVM        0    0    1    0    0    0        0    0    0   0      0    0    1
##           Reference
## Prediction KIPAN KIRC KIRP LGG LIHC LUAD LUSC MESO  OV PAAD PCPG PRAD READ SARC
##   ACC          0    0    0   0    0    0    0    0   0    0    0    0    0    1
##   BLCA         1    0    1   0    0    2    5    0   0    1    0    0    0    1
##   BRCA         0    0    0   0    0    4    2    0   0    0    0    0    0    0
##   CESC         1    1    0   0    0    6   11    1   0    1    0    0    0    0
##   CHOL         0    0    0   0    0    1    0    0   0    2    0    0    0    0
##   COAD         0    0    0   0    0    0    0    0   0    1    0    0    3    0
##   COADREAD     0    0    0   0    0    0    0    0   0    1    0    0   15    0
##   DLBC         0    0    0   0    0    0    0    0   0    1    0    0    0    0
##   ESCA         0    0    0   0    0    0    0    0   0    0    0    1    0    0
##   GBM          2    1    0  18    1    0    0    0   0    1    2    0    0    2
##   GBMLGG       0    0    0  41    0    0    0    0   0    0    0    0    0    0
##   HNSC         0    0    0   0    0    1    8    0   0    0    0    0    0    0
##   KICH        26    6    0   0    0    0    0    0   0    0    0    0    0    0
##   KIPAN       33   24   25   0    0    0    1    0   0    0    0    0    0    0
##   KIRC       128  119    4   0    0    0    0    0   0    0    0    0    0    0
##   KIRP        56    3   54   0    0    0    0    0   0    0    0    0    0    0
##   LGG          0    0    0  94    0    0    0    0   0    0    0    0    0    0
##   LIHC         0    0    0   0  103    0    0    0   0    0    0    0    0    0
##   LUAD         1    0    1   0    0  123    6    0   0    1    0    2    0    0
##   LUSC         1    0    0   0    0    2  109    0   0    0    0    0    0    1
##   MESO         0    0    0   1    0    0    0   20   0    0    0    0    0    2
##   OV           1    0    0   0    0    2    4    0  87    0    0    0    0    0
##   PAAD         0    0    0   0    1    7    1    0   0   40    0    0    1    0
##   PCPG         0    0    0   0    0    0    0    0   0    0   50    0    0    0
##   PRAD         0    0    0   0    0    0    0    0   0    0    0  142    0    0
##   READ         0    0    0   0    0    0    0    0   0    0    0    0    7    0
##   SARC         3    1    1   0    1    1    0    5   0    3    0    3    0   60
##   SKCM         1    0    0   0    0    0    0    0   0    0    0    0    0    0
##   STAD         0    0    0   0    0    2    0    0   0    0    0    0    2    0
##   TGCT         1    0    0   0    0    0    0    0   0    0    0    0    0    0
##   THCA         1    0    0   0    0    0    0    0   0    1    0    0    0    3
##   THYM         0    1    0   0    0    0    1    0   0    0    0    0    0    1
##   UCEC         1    1    0   0    0    2    1    0   1    0    0    1    0    0
##   UCS          5    2    1   0    5    1    1    0   2    0    1    0    0    6
##   UVM          4    0    0   0    0    0    0    0   0    0    0    0    0    0
##           Reference
## Prediction SKCM STAD TGCT THCA THYM UCEC UCS UVM
##   ACC         1    0    0    2    0    0   0   0
##   BLCA        0    0    0    0    0    0   1   0
##   BRCA        0    1    0    1    0    0   0   0
##   CESC        0    2    0    0    0    5   0   0
##   CHOL        0    0    0    0    0    0   0   0
##   COAD        0    1    0    0    0    0   0   0
##   COADREAD    0    5    0    0    0    0   0   0
##   DLBC        0    0    3    1    5    0   0   0
##   ESCA        0    8    0    0    0    0   0   0
##   GBM         0    0    0    0    0    0   1   0
##   GBMLGG      0    0    0    0    0    0   0   0
##   HNSC        3    0    0    0    0    2   0   0
##   KICH        0    0    0    0    0    0   0   0
##   KIPAN       0    0    0    0    0    0   0   0
##   KIRC        0    0    0    0    0    0   0   0
##   KIRP        0    0    0    0    0    0   0   0
##   LGG         0    1    0    0    0    0   0   0
##   LIHC        0    0    0    1    0    0   0   0
##   LUAD        0    1    0    3    0    2   0   0
##   LUSC        0    1    1    0    0    0   0   0
##   MESO        0    0    0    0    0    0   0   0
##   OV          0    1    0    0    1    3   1   0
##   PAAD        0    1    0    0    0    0   0   0
##   PCPG        0    0    0    0    0    0   0   0
##   PRAD        0    0    0    0    0    1   0   0
##   READ        0    1    0    0    0    0   0   0
##   SARC        0    5    0    1    0    0   2   0
##   SKCM       19    0    0    0    0    0   0   3
##   STAD        0   92    0    0    0    0   0   0
##   TGCT        0    3   34    0    0    1   0   0
##   THCA        0    0    0  141    0    0   0   0
##   THYM        1    0    0    0   30    0   0   0
##   UCEC        0    0    0    0    0   37   1   0
##   UCS         2    1    2    0    0    1  11   0
##   UVM         4    0    0    0    0    0   0  21
## 
## Overall Statistics
##                                           
##                Accuracy : 0.6513          
##                  95% CI : (0.6347, 0.6676)
##     No Information Rate : 0.0994          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.6365          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: ACC Class: BLCA Class: BRCA Class: CESC Class: CHOL
## Sensitivity            0.869565     0.66393     0.81040     0.47253   0.3000000
## Specificity            0.998775     0.99274     0.99662     0.98030   0.9990851
## Pos Pred Value         0.833333     0.77885     0.96364     0.40566   0.5000000
## Neg Pred Value         0.999081     0.98713     0.97943     0.98492   0.9978678
## Prevalence             0.006993     0.03709     0.09942     0.02767   0.0030404
## Detection Rate         0.006081     0.02463     0.08057     0.01307   0.0009121
## Detection Prevalence   0.007297     0.03162     0.08361     0.03223   0.0018243
## Balanced Accuracy      0.934170     0.82834     0.90351     0.72641   0.6495425
##                      Class: COAD Class: COADREAD Class: DLBC Class: ESCA
## Sensitivity             0.226190         0.50442    0.642857    0.145455
## Specificity             0.986895         0.97890    0.994198    0.996908
## Pos Pred Value          0.311475         0.45968    0.321429    0.444444
## Neg Pred Value          0.979864         0.98231    0.998467    0.985631
## Prevalence              0.025540         0.03436    0.004257    0.016722
## Detection Rate          0.005777         0.01733    0.002736    0.002432
## Detection Prevalence    0.018547         0.03770    0.008513    0.005473
## Balanced Accuracy       0.606543         0.74166    0.818528    0.571181
##                      Class: GBM Class: GBMLGG Class: HNSC Class: KICH
## Sensitivity             0.88889       0.19000     0.75000    0.842105
## Specificity             0.97102       0.98608     0.97925    0.990214
## Pos Pred Value          0.29851       0.46914     0.64286    0.333333
## Neg Pred Value          0.99842       0.94950     0.98745    0.999074
## Prevalence              0.01368       0.06081     0.04743    0.005777
## Detection Rate          0.01216       0.01155     0.03557    0.004865
## Detection Prevalence    0.04074       0.02463     0.05534    0.014594
## Balanced Accuracy       0.92996       0.58804     0.86463    0.916160
##                      Class: KIPAN Class: KIRC Class: KIRP Class: LGG
## Sensitivity               0.12406     0.74843     0.62069    0.61039
## Specificity               0.98346     0.95783     0.98157    0.96938
## Pos Pred Value            0.39759     0.47410     0.47788    0.49474
## Neg Pred Value            0.92732     0.98683     0.98961    0.98064
## Prevalence                0.08088     0.04834     0.02645    0.04682
## Detection Rate            0.01003     0.03618     0.01642    0.02858
## Detection Prevalence      0.02524     0.07631     0.03436    0.05777
## Balanced Accuracy         0.55376     0.85313     0.80113    0.78988
##                      Class: LIHC Class: LUAD Class: LUSC Class: MESO Class: OV
## Sensitivity              0.92793     0.79870     0.72667    0.769231   0.96667
## Specificity              0.99906     0.99298     0.98216    0.998774   0.99437
## Pos Pred Value           0.97170     0.84828     0.66061    0.833333   0.82857
## Neg Pred Value           0.99749     0.99014     0.98688    0.998162   0.99906
## Prevalence               0.03375     0.04682     0.04561    0.007905   0.02736
## Detection Rate           0.03132     0.03740     0.03314    0.006081   0.02645
## Detection Prevalence     0.03223     0.04409     0.05017    0.007297   0.03192
## Balanced Accuracy        0.96349     0.89584     0.85441    0.884002   0.98052
##                      Class: PAAD Class: PCPG Class: PRAD Class: READ
## Sensitivity              0.75472     0.94340     0.95302    0.250000
## Specificity              0.99289     0.99969     0.99682    0.993254
## Pos Pred Value           0.63492     0.98039     0.93421    0.241379
## Neg Pred Value           0.99597     0.99907     0.99777    0.993558
## Prevalence               0.01611     0.01611     0.04530    0.008513
## Detection Rate           0.01216     0.01520     0.04317    0.002128
## Detection Prevalence     0.01915     0.01551     0.04621    0.008817
## Balanced Accuracy        0.87380     0.97154     0.97492    0.621627
##                      Class: SARC Class: SKCM Class: STAD Class: TGCT
## Sensitivity              0.77922    0.633333     0.74194     0.85000
## Specificity              0.98910    0.997852     0.99084     0.99785
## Pos Pred Value           0.63158    0.730769     0.76033     0.82927
## Neg Pred Value           0.99468    0.996629     0.98990     0.99815
## Prevalence               0.02341    0.009121     0.03770     0.01216
## Detection Rate           0.01824    0.005777     0.02797     0.01034
## Detection Prevalence     0.02888    0.007905     0.03679     0.01247
## Balanced Accuracy        0.88416    0.815593     0.86639     0.92392
##                      Class: THCA Class: THYM Class: UCEC Class: UCS Class: UVM
## Sensitivity              0.94000    0.833333     0.71154   0.647059   0.875000
## Specificity              0.99777    0.996926     0.99042   0.987775   0.996937
## Pos Pred Value           0.95270    0.750000     0.54412   0.215686   0.677419
## Neg Pred Value           0.99713    0.998153     0.99534   0.998147   0.999079
## Prevalence               0.04561    0.010946     0.01581   0.005169   0.007297
## Detection Rate           0.04287    0.009121     0.01125   0.003344   0.006385
## Detection Prevalence     0.04500    0.012162     0.02067   0.015506   0.009425
## Balanced Accuracy        0.96888    0.915130     0.85098   0.817417   0.935969
pheatmap(table(predictions, testing$.id), scale = "column", cluster_rows = FALSE, cluster_cols=FALSE)

# --- collapsing samples by type ---#
mod_id <- gt_all$.id
mod_id <- gsub("^COAD$|^COADREAD$|^READ$", "COLORECTAL", mod_id)
mod_id <- gsub("^LUAD$|^LUSC$", "LUNG", mod_id)
mod_id <- gsub("^UCS$|^UCEC$", "UTERINE", mod_id)
mod_id <- gsub("^PAAD$|^LIHC$|^CHOL$", "PARALIVER", mod_id)
mod_id <- gsub("^ESCA$|^STAD$", "STOPH", mod_id)
mod_id <- gsub("^GBM$|^GBMLGG$|^LGG$", "GLIOMA", mod_id)
mod_id <- gsub("^KICH$|^KIPAN$|^KIRP$|^KIRC$","KIDNEY", mod_id)

gt_all2 = cbind(mod_id, gt_all[,-1])
unique(gt_all2$mod_id)
##  [1] "ACC"        "BLCA"       "BRCA"       "CESC"       "PARALIVER" 
##  [6] "COLORECTAL" "DLBC"       "STOPH"      "GLIOMA"     "HNSC"      
## [11] "KIDNEY"     "LUNG"       "MESO"       "OV"         "PCPG"      
## [16] "PRAD"       "SARC"       "SKCM"       "TGCT"       "THCA"      
## [21] "THYM"       "UTERINE"    "UVM"
confusionMatrix(predictions, as.factor(testing$.id))
## Confusion Matrix and Statistics
## 
##           Reference
## Prediction ACC BLCA BRCA CESC CHOL COAD COADREAD DLBC ESCA GBM GBMLGG HNSC KICH
##   ACC       20    0    0    0    0    0        0    0    0   0      0    0    0
##   BLCA       0   81    4    5    1    0        1    0    0   0      0    0    0
##   BRCA       0    0  265    1    0    0        0    0    0   0      0    1    0
##   CESC       0   17    3   43    0    0        1    0    2   0      0   12    0
##   CHOL       0    0    0    0    3    0        0    0    0   0      0    0    0
##   COAD       0    0    0    0    0   19       35    0    1   0      0    1    0
##   COADREAD   0    0    0    0    0   46       57    0    0   0      0    0    0
##   DLBC       1    0    4    2    0    0        0    9    0   0      0    2    0
##   ESCA       0    0    1    0    0    0        0    0    8   0      0    0    0
##   GBM        0    0    0    0    0    0        0    0    0  40     66    0    0
##   GBMLGG     0    0    0    0    0    0        0    0    0   2     38    0    0
##   HNSC       0   13    4   21    0    0        0    0   13   0      0  117    0
##   KICH       0    0    0    0    0    0        0    0    0   0      0    0   16
##   KIPAN      0    0    0    0    0    0        0    0    0   0      0    0    0
##   KIRC       0    0    0    0    0    0        0    0    0   0      0    0    0
##   KIRP       0    0    0    0    0    0        0    0    0   0      0    0    0
##   LGG        0    0    0    0    0    0        0    0    0   0     95    0    0
##   LIHC       0    0    0    1    0    0        0    1    0   0      0    0    0
##   LUAD       0    0    2    0    2    0        1    0    0   0      0    0    0
##   LUSC       0    6    9    5    0    0        0    0   10   0      0   20    0
##   MESO       0    0    0    0    0    0        0    0    0   0      0    1    0
##   OV         0    0    1    2    1    0        0    0    1   0      0    0    0
##   PAAD       0    0    0    1    2    6        3    0    0   0      0    0    0
##   PCPG       0    0    1    0    0    0        0    0    0   0      0    0    0
##   PRAD       0    0    9    0    0    0        0    0    0   0      0    0    0
##   READ       0    0    0    1    0    9       10    0    1   0      0    0    0
##   SARC       1    1    1    0    0    1        0    0    1   2      1    1    0
##   SKCM       0    0    2    0    0    0        0    0    0   0      0    0    1
##   STAD       0    0    0    0    0    2        5    0   18   0      0    0    0
##   TGCT       0    0    0    0    0    0        0    2    0   0      0    0    0
##   THCA       0    0    2    0    0    0        0    0    0   0      0    0    0
##   THYM       1    1    0    2    1    0        0    1    0   0      0    0    0
##   UCEC       0    1   15    7    0    0        0    0    0   0      0    0    0
##   UCS        0    2    3    0    0    1        0    1    0   1      0    1    1
##   UVM        0    0    1    0    0    0        0    0    0   0      0    0    1
##           Reference
## Prediction KIPAN KIRC KIRP LGG LIHC LUAD LUSC MESO  OV PAAD PCPG PRAD READ SARC
##   ACC          0    0    0   0    0    0    0    0   0    0    0    0    0    1
##   BLCA         1    0    1   0    0    2    5    0   0    1    0    0    0    1
##   BRCA         0    0    0   0    0    4    2    0   0    0    0    0    0    0
##   CESC         1    1    0   0    0    6   11    1   0    1    0    0    0    0
##   CHOL         0    0    0   0    0    1    0    0   0    2    0    0    0    0
##   COAD         0    0    0   0    0    0    0    0   0    1    0    0    3    0
##   COADREAD     0    0    0   0    0    0    0    0   0    1    0    0   15    0
##   DLBC         0    0    0   0    0    0    0    0   0    1    0    0    0    0
##   ESCA         0    0    0   0    0    0    0    0   0    0    0    1    0    0
##   GBM          2    1    0  18    1    0    0    0   0    1    2    0    0    2
##   GBMLGG       0    0    0  41    0    0    0    0   0    0    0    0    0    0
##   HNSC         0    0    0   0    0    1    8    0   0    0    0    0    0    0
##   KICH        26    6    0   0    0    0    0    0   0    0    0    0    0    0
##   KIPAN       33   24   25   0    0    0    1    0   0    0    0    0    0    0
##   KIRC       128  119    4   0    0    0    0    0   0    0    0    0    0    0
##   KIRP        56    3   54   0    0    0    0    0   0    0    0    0    0    0
##   LGG          0    0    0  94    0    0    0    0   0    0    0    0    0    0
##   LIHC         0    0    0   0  103    0    0    0   0    0    0    0    0    0
##   LUAD         1    0    1   0    0  123    6    0   0    1    0    2    0    0
##   LUSC         1    0    0   0    0    2  109    0   0    0    0    0    0    1
##   MESO         0    0    0   1    0    0    0   20   0    0    0    0    0    2
##   OV           1    0    0   0    0    2    4    0  87    0    0    0    0    0
##   PAAD         0    0    0   0    1    7    1    0   0   40    0    0    1    0
##   PCPG         0    0    0   0    0    0    0    0   0    0   50    0    0    0
##   PRAD         0    0    0   0    0    0    0    0   0    0    0  142    0    0
##   READ         0    0    0   0    0    0    0    0   0    0    0    0    7    0
##   SARC         3    1    1   0    1    1    0    5   0    3    0    3    0   60
##   SKCM         1    0    0   0    0    0    0    0   0    0    0    0    0    0
##   STAD         0    0    0   0    0    2    0    0   0    0    0    0    2    0
##   TGCT         1    0    0   0    0    0    0    0   0    0    0    0    0    0
##   THCA         1    0    0   0    0    0    0    0   0    1    0    0    0    3
##   THYM         0    1    0   0    0    0    1    0   0    0    0    0    0    1
##   UCEC         1    1    0   0    0    2    1    0   1    0    0    1    0    0
##   UCS          5    2    1   0    5    1    1    0   2    0    1    0    0    6
##   UVM          4    0    0   0    0    0    0    0   0    0    0    0    0    0
##           Reference
## Prediction SKCM STAD TGCT THCA THYM UCEC UCS UVM
##   ACC         1    0    0    2    0    0   0   0
##   BLCA        0    0    0    0    0    0   1   0
##   BRCA        0    1    0    1    0    0   0   0
##   CESC        0    2    0    0    0    5   0   0
##   CHOL        0    0    0    0    0    0   0   0
##   COAD        0    1    0    0    0    0   0   0
##   COADREAD    0    5    0    0    0    0   0   0
##   DLBC        0    0    3    1    5    0   0   0
##   ESCA        0    8    0    0    0    0   0   0
##   GBM         0    0    0    0    0    0   1   0
##   GBMLGG      0    0    0    0    0    0   0   0
##   HNSC        3    0    0    0    0    2   0   0
##   KICH        0    0    0    0    0    0   0   0
##   KIPAN       0    0    0    0    0    0   0   0
##   KIRC        0    0    0    0    0    0   0   0
##   KIRP        0    0    0    0    0    0   0   0
##   LGG         0    1    0    0    0    0   0   0
##   LIHC        0    0    0    1    0    0   0   0
##   LUAD        0    1    0    3    0    2   0   0
##   LUSC        0    1    1    0    0    0   0   0
##   MESO        0    0    0    0    0    0   0   0
##   OV          0    1    0    0    1    3   1   0
##   PAAD        0    1    0    0    0    0   0   0
##   PCPG        0    0    0    0    0    0   0   0
##   PRAD        0    0    0    0    0    1   0   0
##   READ        0    1    0    0    0    0   0   0
##   SARC        0    5    0    1    0    0   2   0
##   SKCM       19    0    0    0    0    0   0   3
##   STAD        0   92    0    0    0    0   0   0
##   TGCT        0    3   34    0    0    1   0   0
##   THCA        0    0    0  141    0    0   0   0
##   THYM        1    0    0    0   30    0   0   0
##   UCEC        0    0    0    0    0   37   1   0
##   UCS         2    1    2    0    0    1  11   0
##   UVM         4    0    0    0    0    0   0  21
## 
## Overall Statistics
##                                           
##                Accuracy : 0.6513          
##                  95% CI : (0.6347, 0.6676)
##     No Information Rate : 0.0994          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.6365          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: ACC Class: BLCA Class: BRCA Class: CESC Class: CHOL
## Sensitivity            0.869565     0.66393     0.81040     0.47253   0.3000000
## Specificity            0.998775     0.99274     0.99662     0.98030   0.9990851
## Pos Pred Value         0.833333     0.77885     0.96364     0.40566   0.5000000
## Neg Pred Value         0.999081     0.98713     0.97943     0.98492   0.9978678
## Prevalence             0.006993     0.03709     0.09942     0.02767   0.0030404
## Detection Rate         0.006081     0.02463     0.08057     0.01307   0.0009121
## Detection Prevalence   0.007297     0.03162     0.08361     0.03223   0.0018243
## Balanced Accuracy      0.934170     0.82834     0.90351     0.72641   0.6495425
##                      Class: COAD Class: COADREAD Class: DLBC Class: ESCA
## Sensitivity             0.226190         0.50442    0.642857    0.145455
## Specificity             0.986895         0.97890    0.994198    0.996908
## Pos Pred Value          0.311475         0.45968    0.321429    0.444444
## Neg Pred Value          0.979864         0.98231    0.998467    0.985631
## Prevalence              0.025540         0.03436    0.004257    0.016722
## Detection Rate          0.005777         0.01733    0.002736    0.002432
## Detection Prevalence    0.018547         0.03770    0.008513    0.005473
## Balanced Accuracy       0.606543         0.74166    0.818528    0.571181
##                      Class: GBM Class: GBMLGG Class: HNSC Class: KICH
## Sensitivity             0.88889       0.19000     0.75000    0.842105
## Specificity             0.97102       0.98608     0.97925    0.990214
## Pos Pred Value          0.29851       0.46914     0.64286    0.333333
## Neg Pred Value          0.99842       0.94950     0.98745    0.999074
## Prevalence              0.01368       0.06081     0.04743    0.005777
## Detection Rate          0.01216       0.01155     0.03557    0.004865
## Detection Prevalence    0.04074       0.02463     0.05534    0.014594
## Balanced Accuracy       0.92996       0.58804     0.86463    0.916160
##                      Class: KIPAN Class: KIRC Class: KIRP Class: LGG
## Sensitivity               0.12406     0.74843     0.62069    0.61039
## Specificity               0.98346     0.95783     0.98157    0.96938
## Pos Pred Value            0.39759     0.47410     0.47788    0.49474
## Neg Pred Value            0.92732     0.98683     0.98961    0.98064
## Prevalence                0.08088     0.04834     0.02645    0.04682
## Detection Rate            0.01003     0.03618     0.01642    0.02858
## Detection Prevalence      0.02524     0.07631     0.03436    0.05777
## Balanced Accuracy         0.55376     0.85313     0.80113    0.78988
##                      Class: LIHC Class: LUAD Class: LUSC Class: MESO Class: OV
## Sensitivity              0.92793     0.79870     0.72667    0.769231   0.96667
## Specificity              0.99906     0.99298     0.98216    0.998774   0.99437
## Pos Pred Value           0.97170     0.84828     0.66061    0.833333   0.82857
## Neg Pred Value           0.99749     0.99014     0.98688    0.998162   0.99906
## Prevalence               0.03375     0.04682     0.04561    0.007905   0.02736
## Detection Rate           0.03132     0.03740     0.03314    0.006081   0.02645
## Detection Prevalence     0.03223     0.04409     0.05017    0.007297   0.03192
## Balanced Accuracy        0.96349     0.89584     0.85441    0.884002   0.98052
##                      Class: PAAD Class: PCPG Class: PRAD Class: READ
## Sensitivity              0.75472     0.94340     0.95302    0.250000
## Specificity              0.99289     0.99969     0.99682    0.993254
## Pos Pred Value           0.63492     0.98039     0.93421    0.241379
## Neg Pred Value           0.99597     0.99907     0.99777    0.993558
## Prevalence               0.01611     0.01611     0.04530    0.008513
## Detection Rate           0.01216     0.01520     0.04317    0.002128
## Detection Prevalence     0.01915     0.01551     0.04621    0.008817
## Balanced Accuracy        0.87380     0.97154     0.97492    0.621627
##                      Class: SARC Class: SKCM Class: STAD Class: TGCT
## Sensitivity              0.77922    0.633333     0.74194     0.85000
## Specificity              0.98910    0.997852     0.99084     0.99785
## Pos Pred Value           0.63158    0.730769     0.76033     0.82927
## Neg Pred Value           0.99468    0.996629     0.98990     0.99815
## Prevalence               0.02341    0.009121     0.03770     0.01216
## Detection Rate           0.01824    0.005777     0.02797     0.01034
## Detection Prevalence     0.02888    0.007905     0.03679     0.01247
## Balanced Accuracy        0.88416    0.815593     0.86639     0.92392
##                      Class: THCA Class: THYM Class: UCEC Class: UCS Class: UVM
## Sensitivity              0.94000    0.833333     0.71154   0.647059   0.875000
## Specificity              0.99777    0.996926     0.99042   0.987775   0.996937
## Pos Pred Value           0.95270    0.750000     0.54412   0.215686   0.677419
## Neg Pred Value           0.99713    0.998153     0.99534   0.998147   0.999079
## Prevalence               0.04561    0.010946     0.01581   0.005169   0.007297
## Detection Rate           0.04287    0.009121     0.01125   0.003344   0.006385
## Detection Prevalence     0.04500    0.012162     0.02067   0.015506   0.009425
## Balanced Accuracy        0.96888    0.915130     0.85098   0.817417   0.935969
# --- preprocessing ---#
set.seed(1234)
intrain  = createDataPartition(y = mod_id, p = 0.7, list = FALSE)
training = gt_all2[intrain,]
testing  = gt_all2[-intrain,]
set.seed(1234)
pp       = preProcess(training, method = c("nzv", "scale", "center", "YeoJohnson"))
pptraining   = predict(pp, training)
pptesting = predict(pp, testing)
# --- I think this should stay to show that rda was picked based on performance, no bias ---#
# note - glioma among best performing 
# --- algorithm/method choice ---#
set.seed(123456)
#methods = c("pam", "knn", "rf", "xgbTree", "lda", "rda", "xgbTree", "lda")
methods = c("rda")
measures = vector(length = length(methods), mode = "list")
for (i in 1:length(methods)) {
    method = methods[i]
    print(method)
    model.fit   = train(mod_id ~ ., data = pptraining, method = method)
    predictions = predict(model.fit, pptesting[,-1])
    confmat = confusionMatrix(predictions, as.factor(pptesting$mod_id))
    print(confmat)
    pheatmap(table(predictions, pptesting$mod_id), scale = "column", cluster_rows=FALSE, cluster_cols=FALSE)
    names(measures)[i] = method
    measures[[i]] = confmat$byClass
}
## [1] "rda"
## Confusion Matrix and Statistics
## 
##             Reference
## Prediction   ACC BLCA BRCA CESC COLORECTAL DLBC GLIOMA HNSC KIDNEY LUNG MESO
##   ACC         22    0    0    0          0    0      0    0      0    0    0
##   BLCA         0  100    2    2          0    0      0    1      3    4    0
##   BRCA         0    1  319    0          0    0      0    0      0    0    0
##   CESC         0    3    0   71          2    0      0    4      0    6    0
##   COLORECTAL   0    0    0    1        221    0      0    0      0    0    0
##   DLBC         0    0    0    0          0   14      0    0      0    0    0
##   GLIOMA       0    0    0    0          0    0    398    0      0    0    0
##   HNSC         0    8    2    8          0    0      0  134      0   20    0
##   KIDNEY       0    0    0    0          0    0      0    0    528    0    0
##   LUNG         0    6    1    3          0    0      0   14      0  262    0
##   MESO         0    0    0    0          0    0      0    0      0    0   25
##   OV           0    0    0    0          0    0      0    0      0    2    0
##   PARALIVER    0    0    0    0          0    0      0    0      0    1    0
##   PCPG         0    0    0    0          0    0      0    0      0    1    0
##   PRAD         0    0    0    0          0    0      0    0      0    0    0
##   SARC         1    2    3    0          0    0      2    2      2    1    1
##   SKCM         0    0    0    0          0    0      0    0      0    0    0
##   STOPH        0    1    0    2          3    0      0    0      0    5    0
##   TGCT         0    0    0    0          0    0      0    0      0    0    0
##   THCA         0    0    0    0          0    0      0    0      0    0    0
##   THYM         0    0    0    0          0    0      0    0      0    0    0
##   UTERINE      0    1    0    4          0    0      0    1      0    2    0
##   UVM          0    0    0    0          0    0      0    0      0    0    0
##             Reference
## Prediction    OV PARALIVER PCPG PRAD SARC SKCM STOPH TGCT THCA THYM UTERINE UVM
##   ACC          0         0    0    0    0    0     0    0    0    0       1   0
##   BLCA         0         0    0    0    0    0     0    0    0    0       0   0
##   BRCA         0         0    0    0    0    0     0    0    0    0       0   0
##   CESC         0         0    0    0    0    0     0    0    0    0       0   0
##   COLORECTAL   0         3    0    0    0    0     6    0    0    0       0   0
##   DLBC         0         0    0    0    0    0     1    0    0    0       0   0
##   GLIOMA       0         0    0    0    0    0     0    0    0    0       0   0
##   HNSC         0         0    0    0    0    0     9    0    0    1       0   0
##   KIDNEY       0         1    0    0    0    0     0    0    0    0       0   0
##   LUNG         0         2    0    0    0    0     7    0    0    0       1   0
##   MESO         0         0    0    0    0    0     0    0    0    0       0   0
##   OV          85         0    0    0    0    0     0    0    0    0       4   0
##   PARALIVER    0       165    0    0    0    0     0    1    0    0       0   0
##   PCPG         0         0   53    0    0    0     0    0    0    0       0   0
##   PRAD         0         0    0  149    0    0     0    0    0    0       0   0
##   SARC         1         3    0    0   75    0     0    0    0    0       3   0
##   SKCM         0         0    0    0    1   29     0    0    0    0       0   0
##   STOPH        0         1    0    0    0    0   156    0    0    0       0   0
##   TGCT         0         0    0    0    0    0     0   38    0    0       0   0
##   THCA         0         0    0    0    0    0     0    0  150    0       0   0
##   THYM         0         0    0    0    0    0     0    0    0   34       0   0
##   UTERINE      4         0    0    0    1    0     0    1    0    1      60   0
##   UVM          0         0    0    0    0    1     0    0    0    0       0  24
## 
## Overall Statistics
##                                           
##                Accuracy : 0.9447          
##                  95% CI : (0.9364, 0.9523)
##     No Information Rate : 0.1618          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.9399          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: ACC Class: BLCA Class: BRCA Class: CESC
## Sensitivity            0.956522     0.81967     0.97554     0.78022
## Specificity            0.999694     0.99622     0.99966     0.99532
## Pos Pred Value         0.956522     0.89286     0.99687     0.82558
## Neg Pred Value         0.999694     0.99309     0.99731     0.99377
## Prevalence             0.006982     0.03704     0.09927     0.02763
## Detection Rate         0.006679     0.03036     0.09684     0.02155
## Detection Prevalence   0.006982     0.03400     0.09715     0.02611
## Balanced Accuracy      0.978108     0.90794     0.98760     0.88777
##                      Class: COLORECTAL Class: DLBC Class: GLIOMA Class: HNSC
## Sensitivity                    0.97788    1.000000        0.9950     0.85897
## Specificity                    0.99674    0.999695        1.0000     0.98470
## Pos Pred Value                 0.95671    0.933333        1.0000     0.73626
## Neg Pred Value                 0.99837    1.000000        0.9993     0.99293
## Prevalence                     0.06861    0.004250        0.1214     0.04736
## Detection Rate                 0.06709    0.004250        0.1208     0.04068
## Detection Prevalence           0.07013    0.004554        0.1208     0.05525
## Balanced Accuracy              0.98731    0.999848        0.9975     0.92184
##                      Class: KIDNEY Class: LUNG Class: MESO Class: OV
## Sensitivity                 0.9906     0.86184    0.961538   0.94444
## Specificity                 0.9996     0.98863    1.000000   0.99813
## Pos Pred Value              0.9981     0.88514    1.000000   0.93407
## Neg Pred Value              0.9982     0.98599    0.999694   0.99844
## Prevalence                  0.1618     0.09229    0.007893   0.02732
## Detection Rate              0.1603     0.07954    0.007590   0.02580
## Detection Prevalence        0.1606     0.08986    0.007590   0.02763
## Balanced Accuracy           0.9951     0.92524    0.980769   0.97129
##                      Class: PARALIVER Class: PCPG Class: PRAD Class: SARC
## Sensitivity                   0.94286     1.00000     1.00000     0.97403
## Specificity                   0.99936     0.99969     1.00000     0.99347
## Pos Pred Value                0.98802     0.98148     1.00000     0.78125
## Neg Pred Value                0.99680     1.00000     1.00000     0.99937
## Prevalence                    0.05313     0.01609     0.04523     0.02338
## Detection Rate                0.05009     0.01609     0.04523     0.02277
## Detection Prevalence          0.05070     0.01639     0.04523     0.02914
## Balanced Accuracy             0.97111     0.99985     1.00000     0.98375
##                      Class: SKCM Class: STOPH Class: TGCT Class: THCA
## Sensitivity             0.966667      0.87151     0.95000     1.00000
## Specificity             0.999694      0.99615     1.00000     1.00000
## Pos Pred Value          0.966667      0.92857     1.00000     1.00000
## Neg Pred Value          0.999694      0.99264     0.99939     1.00000
## Prevalence              0.009107      0.05434     0.01214     0.04554
## Detection Rate          0.008804      0.04736     0.01154     0.04554
## Detection Prevalence    0.009107      0.05100     0.01154     0.04554
## Balanced Accuracy       0.983180      0.93383     0.97500     1.00000
##                      Class: THYM Class: UTERINE Class: UVM
## Sensitivity              0.94444        0.86957   1.000000
## Specificity              1.00000        0.99535   0.999694
## Pos Pred Value           1.00000        0.80000   0.960000
## Neg Pred Value           0.99939        0.99720   1.000000
## Prevalence               0.01093        0.02095   0.007286
## Detection Rate           0.01032        0.01821   0.007286
## Detection Prevalence     0.01032        0.02277   0.007590
## Balanced Accuracy        0.97222        0.93246   0.999847

print(confmat)
## Confusion Matrix and Statistics
## 
##             Reference
## Prediction   ACC BLCA BRCA CESC COLORECTAL DLBC GLIOMA HNSC KIDNEY LUNG MESO
##   ACC         22    0    0    0          0    0      0    0      0    0    0
##   BLCA         0  100    2    2          0    0      0    1      3    4    0
##   BRCA         0    1  319    0          0    0      0    0      0    0    0
##   CESC         0    3    0   71          2    0      0    4      0    6    0
##   COLORECTAL   0    0    0    1        221    0      0    0      0    0    0
##   DLBC         0    0    0    0          0   14      0    0      0    0    0
##   GLIOMA       0    0    0    0          0    0    398    0      0    0    0
##   HNSC         0    8    2    8          0    0      0  134      0   20    0
##   KIDNEY       0    0    0    0          0    0      0    0    528    0    0
##   LUNG         0    6    1    3          0    0      0   14      0  262    0
##   MESO         0    0    0    0          0    0      0    0      0    0   25
##   OV           0    0    0    0          0    0      0    0      0    2    0
##   PARALIVER    0    0    0    0          0    0      0    0      0    1    0
##   PCPG         0    0    0    0          0    0      0    0      0    1    0
##   PRAD         0    0    0    0          0    0      0    0      0    0    0
##   SARC         1    2    3    0          0    0      2    2      2    1    1
##   SKCM         0    0    0    0          0    0      0    0      0    0    0
##   STOPH        0    1    0    2          3    0      0    0      0    5    0
##   TGCT         0    0    0    0          0    0      0    0      0    0    0
##   THCA         0    0    0    0          0    0      0    0      0    0    0
##   THYM         0    0    0    0          0    0      0    0      0    0    0
##   UTERINE      0    1    0    4          0    0      0    1      0    2    0
##   UVM          0    0    0    0          0    0      0    0      0    0    0
##             Reference
## Prediction    OV PARALIVER PCPG PRAD SARC SKCM STOPH TGCT THCA THYM UTERINE UVM
##   ACC          0         0    0    0    0    0     0    0    0    0       1   0
##   BLCA         0         0    0    0    0    0     0    0    0    0       0   0
##   BRCA         0         0    0    0    0    0     0    0    0    0       0   0
##   CESC         0         0    0    0    0    0     0    0    0    0       0   0
##   COLORECTAL   0         3    0    0    0    0     6    0    0    0       0   0
##   DLBC         0         0    0    0    0    0     1    0    0    0       0   0
##   GLIOMA       0         0    0    0    0    0     0    0    0    0       0   0
##   HNSC         0         0    0    0    0    0     9    0    0    1       0   0
##   KIDNEY       0         1    0    0    0    0     0    0    0    0       0   0
##   LUNG         0         2    0    0    0    0     7    0    0    0       1   0
##   MESO         0         0    0    0    0    0     0    0    0    0       0   0
##   OV          85         0    0    0    0    0     0    0    0    0       4   0
##   PARALIVER    0       165    0    0    0    0     0    1    0    0       0   0
##   PCPG         0         0   53    0    0    0     0    0    0    0       0   0
##   PRAD         0         0    0  149    0    0     0    0    0    0       0   0
##   SARC         1         3    0    0   75    0     0    0    0    0       3   0
##   SKCM         0         0    0    0    1   29     0    0    0    0       0   0
##   STOPH        0         1    0    0    0    0   156    0    0    0       0   0
##   TGCT         0         0    0    0    0    0     0   38    0    0       0   0
##   THCA         0         0    0    0    0    0     0    0  150    0       0   0
##   THYM         0         0    0    0    0    0     0    0    0   34       0   0
##   UTERINE      4         0    0    0    1    0     0    1    0    1      60   0
##   UVM          0         0    0    0    0    1     0    0    0    0       0  24
## 
## Overall Statistics
##                                           
##                Accuracy : 0.9447          
##                  95% CI : (0.9364, 0.9523)
##     No Information Rate : 0.1618          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.9399          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: ACC Class: BLCA Class: BRCA Class: CESC
## Sensitivity            0.956522     0.81967     0.97554     0.78022
## Specificity            0.999694     0.99622     0.99966     0.99532
## Pos Pred Value         0.956522     0.89286     0.99687     0.82558
## Neg Pred Value         0.999694     0.99309     0.99731     0.99377
## Prevalence             0.006982     0.03704     0.09927     0.02763
## Detection Rate         0.006679     0.03036     0.09684     0.02155
## Detection Prevalence   0.006982     0.03400     0.09715     0.02611
## Balanced Accuracy      0.978108     0.90794     0.98760     0.88777
##                      Class: COLORECTAL Class: DLBC Class: GLIOMA Class: HNSC
## Sensitivity                    0.97788    1.000000        0.9950     0.85897
## Specificity                    0.99674    0.999695        1.0000     0.98470
## Pos Pred Value                 0.95671    0.933333        1.0000     0.73626
## Neg Pred Value                 0.99837    1.000000        0.9993     0.99293
## Prevalence                     0.06861    0.004250        0.1214     0.04736
## Detection Rate                 0.06709    0.004250        0.1208     0.04068
## Detection Prevalence           0.07013    0.004554        0.1208     0.05525
## Balanced Accuracy              0.98731    0.999848        0.9975     0.92184
##                      Class: KIDNEY Class: LUNG Class: MESO Class: OV
## Sensitivity                 0.9906     0.86184    0.961538   0.94444
## Specificity                 0.9996     0.98863    1.000000   0.99813
## Pos Pred Value              0.9981     0.88514    1.000000   0.93407
## Neg Pred Value              0.9982     0.98599    0.999694   0.99844
## Prevalence                  0.1618     0.09229    0.007893   0.02732
## Detection Rate              0.1603     0.07954    0.007590   0.02580
## Detection Prevalence        0.1606     0.08986    0.007590   0.02763
## Balanced Accuracy           0.9951     0.92524    0.980769   0.97129
##                      Class: PARALIVER Class: PCPG Class: PRAD Class: SARC
## Sensitivity                   0.94286     1.00000     1.00000     0.97403
## Specificity                   0.99936     0.99969     1.00000     0.99347
## Pos Pred Value                0.98802     0.98148     1.00000     0.78125
## Neg Pred Value                0.99680     1.00000     1.00000     0.99937
## Prevalence                    0.05313     0.01609     0.04523     0.02338
## Detection Rate                0.05009     0.01609     0.04523     0.02277
## Detection Prevalence          0.05070     0.01639     0.04523     0.02914
## Balanced Accuracy             0.97111     0.99985     1.00000     0.98375
##                      Class: SKCM Class: STOPH Class: TGCT Class: THCA
## Sensitivity             0.966667      0.87151     0.95000     1.00000
## Specificity             0.999694      0.99615     1.00000     1.00000
## Pos Pred Value          0.966667      0.92857     1.00000     1.00000
## Neg Pred Value          0.999694      0.99264     0.99939     1.00000
## Prevalence              0.009107      0.05434     0.01214     0.04554
## Detection Rate          0.008804      0.04736     0.01154     0.04554
## Detection Prevalence    0.009107      0.05100     0.01154     0.04554
## Balanced Accuracy       0.983180      0.93383     0.97500     1.00000
##                      Class: THYM Class: UTERINE Class: UVM
## Sensitivity              0.94444        0.86957   1.000000
## Specificity              1.00000        0.99535   0.999694
## Pos Pred Value           1.00000        0.80000   0.960000
## Neg Pred Value           0.99939        0.99720   1.000000
## Prevalence               0.01093        0.02095   0.007286
## Detection Rate           0.01032        0.01821   0.007286
## Detection Prevalence     0.01032        0.02277   0.007590
## Balanced Accuracy        0.97222        0.93246   0.999847
measuresDF = ldply(measures, data.frame)
# --- method comparison ---#
type = unique(mod_id)
measuresDF$type = rep(type, length(unique(measuresDF$.id)))
measuresDFm = melt(measuresDF)
measuresDFm = dcast(measuresDFm, .id + variable ~ type)
measuresDFm = measuresDFm[-grep("Prevalence|Detection", measuresDFm$variable),]
measuresList = split(measuresDFm, as.character(measuresDFm$variable))

for (i in 1:length(measuresList)) {
    print(names(measuresList)[i])
    print(measuresList[[i]])
    print(ggradar(measuresList[[i]][,-2]))
}
## [1] "Balanced.Accuracy"
##    .id          variable      ACC      BLCA      BRCA      CESC COLORECTAL
## 11 rda Balanced.Accuracy 0.978108 0.9079445 0.9875991 0.8877683  0.9998476
##      DLBC    GLIOMA      HNSC    KIDNEY      LUNG     MESO        OV PARALIVER
## 11 0.9975 0.9951285 0.9252354 0.9807692 0.9712859 0.971108 0.9998457 0.9873083
##    PCPG      PRAD      SARC     SKCM    STOPH  TGCT THCA      THYM  UTERINE
## 11    1 0.9837491 0.9831801 0.933828 0.921839 0.975    1 0.9722222 0.932457
##          UVM
## 11 0.9998471

## [1] "F1"
##   .id variable       ACC      BLCA      BRCA      CESC COLORECTAL      DLBC
## 7 rda       F1 0.9565217 0.8547009 0.9860896 0.8022599  0.9655172 0.9974937
##      GLIOMA      HNSC    KIDNEY      LUNG      MESO        OV PARALIVER PCPG
## 7 0.9943503 0.8733333 0.9803922 0.9392265 0.9649123 0.9906542 0.9671772    1
##       PRAD      SARC      SKCM     STOPH     TGCT THCA      THYM   UTERINE
## 7 0.867052 0.9666667 0.8991354 0.7928994 0.974359    1 0.9714286 0.8333333
##         UVM
## 7 0.9795918

## [1] "Neg.Pred.Value"
##   .id       variable       ACC      BLCA    BRCA      CESC COLORECTAL      DLBC
## 4 rda Neg.Pred.Value 0.9996943 0.9930861 0.99731 0.9937656          1 0.9993094
##      GLIOMA      HNSC    KIDNEY     LUNG     MESO OV PARALIVER PCPG      PRAD
## 4 0.9981917 0.9859907 0.9996941 0.998439 0.996802  1 0.9983676    1 0.9993746
##        SARC      SKCM     STOPH      TGCT THCA      THYM   UTERINE UVM
## 4 0.9996936 0.9926424 0.9929306 0.9993857    1 0.9993865 0.9972041   1

## [1] "Pos.Pred.Value"
##   .id       variable       ACC      BLCA     BRCA      CESC COLORECTAL DLBC
## 3 rda Pos.Pred.Value 0.9565217 0.8928571 0.996875 0.8255814  0.9333333    1
##      GLIOMA      HNSC KIDNEY      LUNG     MESO        OV PARALIVER PCPG
## 3 0.9981096 0.8851351      1 0.9340659 0.988024 0.9814815   0.95671    1
##      PRAD      SARC      SKCM     STOPH TGCT THCA THYM UTERINE  UVM
## 3 0.78125 0.9666667 0.9285714 0.7362637    1    1    1     0.8 0.96

## [1] "Precision"
##   .id  variable       ACC      BLCA     BRCA      CESC COLORECTAL DLBC
## 5 rda Precision 0.9565217 0.8928571 0.996875 0.8255814  0.9333333    1
##      GLIOMA      HNSC KIDNEY      LUNG     MESO        OV PARALIVER PCPG
## 5 0.9981096 0.8851351      1 0.9340659 0.988024 0.9814815   0.95671    1
##      PRAD      SARC      SKCM     STOPH TGCT THCA THYM UTERINE  UVM
## 5 0.78125 0.9666667 0.9285714 0.7362637    1    1    1     0.8 0.96

## [1] "Recall"
##   .id variable       ACC      BLCA      BRCA      CESC COLORECTAL  DLBC
## 6 rda   Recall 0.9565217 0.8196721 0.9755352 0.7802198          1 0.995
##      GLIOMA      HNSC    KIDNEY      LUNG      MESO OV PARALIVER PCPG     PRAD
## 6 0.9906191 0.8618421 0.9615385 0.9444444 0.9428571  1 0.9778761    1 0.974026
##        SARC      SKCM     STOPH TGCT THCA      THYM   UTERINE UVM
## 6 0.9666667 0.8715084 0.8589744 0.95    1 0.9444444 0.8695652   1

## [1] "Sensitivity"
##   .id    variable       ACC      BLCA      BRCA      CESC COLORECTAL  DLBC
## 1 rda Sensitivity 0.9565217 0.8196721 0.9755352 0.7802198          1 0.995
##      GLIOMA      HNSC    KIDNEY      LUNG      MESO OV PARALIVER PCPG     PRAD
## 1 0.9906191 0.8618421 0.9615385 0.9444444 0.9428571  1 0.9778761    1 0.974026
##        SARC      SKCM     STOPH TGCT THCA      THYM   UTERINE UVM
## 1 0.9666667 0.8715084 0.8589744 0.95    1 0.9444444 0.8695652   1

## [1] "Specificity"
##   .id    variable       ACC      BLCA     BRCA      CESC COLORECTAL DLBC
## 2 rda Specificity 0.9996943 0.9962169 0.999663 0.9953169  0.9996951    1
##      GLIOMA      HNSC KIDNEY      LUNG      MESO        OV PARALIVER PCPG
## 2 0.9996378 0.9886288      1 0.9981273 0.9993588 0.9996915 0.9967405    1
##        PRAD      SARC      SKCM     STOPH TGCT THCA THYM   UTERINE       UVM
## 2 0.9934722 0.9996936 0.9961477 0.9847036    1    1    1 0.9953488 0.9996942

ctrl = trainControl(method = "repeatedcv",                # repeated K-folds
                    number = 10,                          # 10 folds
                    repeats = 10,                         # 10 repeats
                    summaryFunction = multiClassSummary,  # Evaluate Performance 
                    classProbs = T,                       # Estimate class probabilities
                    savePredictions = T,
                    verboseIter = T)
rdaGrid = data.frame(gamma = (1:4)/4, lambda = 0.75)
set.seed(1234)
rdafit = train(mod_id ~ ., data = pptraining, method = "rda", tuneGrid = rdaGrid, trControl = ctrl)
## + Fold01.Rep01: gamma=0.25, lambda=0.75 
## - Fold01.Rep01: gamma=0.25, lambda=0.75 
## + Fold01.Rep01: gamma=0.50, lambda=0.75 
## - Fold01.Rep01: gamma=0.50, lambda=0.75 
## + Fold01.Rep01: gamma=0.75, lambda=0.75 
## - Fold01.Rep01: gamma=0.75, lambda=0.75 
## + Fold01.Rep01: gamma=1.00, lambda=0.75 
## - Fold01.Rep01: gamma=1.00, lambda=0.75 
## + Fold02.Rep01: gamma=0.25, lambda=0.75 
## - Fold02.Rep01: gamma=0.25, lambda=0.75 
## + Fold02.Rep01: gamma=0.50, lambda=0.75 
## - Fold02.Rep01: gamma=0.50, lambda=0.75 
## + Fold02.Rep01: gamma=0.75, lambda=0.75 
## - Fold02.Rep01: gamma=0.75, lambda=0.75 
## + Fold02.Rep01: gamma=1.00, lambda=0.75 
## - Fold02.Rep01: gamma=1.00, lambda=0.75 
## + Fold03.Rep01: gamma=0.25, lambda=0.75 
## - Fold03.Rep01: gamma=0.25, lambda=0.75 
## + Fold03.Rep01: gamma=0.50, lambda=0.75 
## - Fold03.Rep01: gamma=0.50, lambda=0.75 
## + Fold03.Rep01: gamma=0.75, lambda=0.75 
## - Fold03.Rep01: gamma=0.75, lambda=0.75 
## + Fold03.Rep01: gamma=1.00, lambda=0.75 
## - Fold03.Rep01: gamma=1.00, lambda=0.75 
## + Fold04.Rep01: gamma=0.25, lambda=0.75 
## - Fold04.Rep01: gamma=0.25, lambda=0.75 
## + Fold04.Rep01: gamma=0.50, lambda=0.75 
## - Fold04.Rep01: gamma=0.50, lambda=0.75 
## + Fold04.Rep01: gamma=0.75, lambda=0.75 
## - Fold04.Rep01: gamma=0.75, lambda=0.75 
## + Fold04.Rep01: gamma=1.00, lambda=0.75 
## - Fold04.Rep01: gamma=1.00, lambda=0.75 
## + Fold05.Rep01: gamma=0.25, lambda=0.75 
## - Fold05.Rep01: gamma=0.25, lambda=0.75 
## + Fold05.Rep01: gamma=0.50, lambda=0.75 
## - Fold05.Rep01: gamma=0.50, lambda=0.75 
## + Fold05.Rep01: gamma=0.75, lambda=0.75 
## - Fold05.Rep01: gamma=0.75, lambda=0.75 
## + Fold05.Rep01: gamma=1.00, lambda=0.75 
## - Fold05.Rep01: gamma=1.00, lambda=0.75 
## + Fold06.Rep01: gamma=0.25, lambda=0.75 
## - Fold06.Rep01: gamma=0.25, lambda=0.75 
## + Fold06.Rep01: gamma=0.50, lambda=0.75 
## - Fold06.Rep01: gamma=0.50, lambda=0.75 
## + Fold06.Rep01: gamma=0.75, lambda=0.75 
## - Fold06.Rep01: gamma=0.75, lambda=0.75 
## + Fold06.Rep01: gamma=1.00, lambda=0.75 
## - Fold06.Rep01: gamma=1.00, lambda=0.75 
## + Fold07.Rep01: gamma=0.25, lambda=0.75 
## - Fold07.Rep01: gamma=0.25, lambda=0.75 
## + Fold07.Rep01: gamma=0.50, lambda=0.75 
## - Fold07.Rep01: gamma=0.50, lambda=0.75 
## + Fold07.Rep01: gamma=0.75, lambda=0.75 
## - Fold07.Rep01: gamma=0.75, lambda=0.75 
## + Fold07.Rep01: gamma=1.00, lambda=0.75 
## - Fold07.Rep01: gamma=1.00, lambda=0.75 
## + Fold08.Rep01: gamma=0.25, lambda=0.75 
## - Fold08.Rep01: gamma=0.25, lambda=0.75 
## + Fold08.Rep01: gamma=0.50, lambda=0.75 
## - Fold08.Rep01: gamma=0.50, lambda=0.75 
## + Fold08.Rep01: gamma=0.75, lambda=0.75 
## - Fold08.Rep01: gamma=0.75, lambda=0.75 
## + Fold08.Rep01: gamma=1.00, lambda=0.75 
## - Fold08.Rep01: gamma=1.00, lambda=0.75 
## + Fold09.Rep01: gamma=0.25, lambda=0.75 
## - Fold09.Rep01: gamma=0.25, lambda=0.75 
## + Fold09.Rep01: gamma=0.50, lambda=0.75 
## - Fold09.Rep01: gamma=0.50, lambda=0.75 
## + Fold09.Rep01: gamma=0.75, lambda=0.75 
## - Fold09.Rep01: gamma=0.75, lambda=0.75 
## + Fold09.Rep01: gamma=1.00, lambda=0.75 
## - Fold09.Rep01: gamma=1.00, lambda=0.75 
## + Fold10.Rep01: gamma=0.25, lambda=0.75 
## - Fold10.Rep01: gamma=0.25, lambda=0.75 
## + Fold10.Rep01: gamma=0.50, lambda=0.75 
## - Fold10.Rep01: gamma=0.50, lambda=0.75 
## + Fold10.Rep01: gamma=0.75, lambda=0.75 
## - Fold10.Rep01: gamma=0.75, lambda=0.75 
## + Fold10.Rep01: gamma=1.00, lambda=0.75 
## - Fold10.Rep01: gamma=1.00, lambda=0.75 
## + Fold01.Rep02: gamma=0.25, lambda=0.75 
## - Fold01.Rep02: gamma=0.25, lambda=0.75 
## + Fold01.Rep02: gamma=0.50, lambda=0.75 
## - Fold01.Rep02: gamma=0.50, lambda=0.75 
## + Fold01.Rep02: gamma=0.75, lambda=0.75 
## - Fold01.Rep02: gamma=0.75, lambda=0.75 
## + Fold01.Rep02: gamma=1.00, lambda=0.75 
## - Fold01.Rep02: gamma=1.00, lambda=0.75 
## + Fold02.Rep02: gamma=0.25, lambda=0.75 
## - Fold02.Rep02: gamma=0.25, lambda=0.75 
## + Fold02.Rep02: gamma=0.50, lambda=0.75 
## - Fold02.Rep02: gamma=0.50, lambda=0.75 
## + Fold02.Rep02: gamma=0.75, lambda=0.75 
## - Fold02.Rep02: gamma=0.75, lambda=0.75 
## + Fold02.Rep02: gamma=1.00, lambda=0.75 
## - Fold02.Rep02: gamma=1.00, lambda=0.75 
## + Fold03.Rep02: gamma=0.25, lambda=0.75 
## - Fold03.Rep02: gamma=0.25, lambda=0.75 
## + Fold03.Rep02: gamma=0.50, lambda=0.75 
## - Fold03.Rep02: gamma=0.50, lambda=0.75 
## + Fold03.Rep02: gamma=0.75, lambda=0.75 
## - Fold03.Rep02: gamma=0.75, lambda=0.75 
## + Fold03.Rep02: gamma=1.00, lambda=0.75 
## - Fold03.Rep02: gamma=1.00, lambda=0.75 
## + Fold04.Rep02: gamma=0.25, lambda=0.75 
## - Fold04.Rep02: gamma=0.25, lambda=0.75 
## + Fold04.Rep02: gamma=0.50, lambda=0.75 
## - Fold04.Rep02: gamma=0.50, lambda=0.75 
## + Fold04.Rep02: gamma=0.75, lambda=0.75 
## - Fold04.Rep02: gamma=0.75, lambda=0.75 
## + Fold04.Rep02: gamma=1.00, lambda=0.75 
## - Fold04.Rep02: gamma=1.00, lambda=0.75 
## + Fold05.Rep02: gamma=0.25, lambda=0.75 
## - Fold05.Rep02: gamma=0.25, lambda=0.75 
## + Fold05.Rep02: gamma=0.50, lambda=0.75 
## - Fold05.Rep02: gamma=0.50, lambda=0.75 
## + Fold05.Rep02: gamma=0.75, lambda=0.75 
## - Fold05.Rep02: gamma=0.75, lambda=0.75 
## + Fold05.Rep02: gamma=1.00, lambda=0.75 
## - Fold05.Rep02: gamma=1.00, lambda=0.75 
## + Fold06.Rep02: gamma=0.25, lambda=0.75 
## - Fold06.Rep02: gamma=0.25, lambda=0.75 
## + Fold06.Rep02: gamma=0.50, lambda=0.75 
## - Fold06.Rep02: gamma=0.50, lambda=0.75 
## + Fold06.Rep02: gamma=0.75, lambda=0.75 
## - Fold06.Rep02: gamma=0.75, lambda=0.75 
## + Fold06.Rep02: gamma=1.00, lambda=0.75 
## - Fold06.Rep02: gamma=1.00, lambda=0.75 
## + Fold07.Rep02: gamma=0.25, lambda=0.75 
## - Fold07.Rep02: gamma=0.25, lambda=0.75 
## + Fold07.Rep02: gamma=0.50, lambda=0.75 
## - Fold07.Rep02: gamma=0.50, lambda=0.75 
## + Fold07.Rep02: gamma=0.75, lambda=0.75 
## - Fold07.Rep02: gamma=0.75, lambda=0.75 
## + Fold07.Rep02: gamma=1.00, lambda=0.75 
## - Fold07.Rep02: gamma=1.00, lambda=0.75 
## + Fold08.Rep02: gamma=0.25, lambda=0.75 
## - Fold08.Rep02: gamma=0.25, lambda=0.75 
## + Fold08.Rep02: gamma=0.50, lambda=0.75 
## - Fold08.Rep02: gamma=0.50, lambda=0.75 
## + Fold08.Rep02: gamma=0.75, lambda=0.75 
## - Fold08.Rep02: gamma=0.75, lambda=0.75 
## + Fold08.Rep02: gamma=1.00, lambda=0.75 
## - Fold08.Rep02: gamma=1.00, lambda=0.75 
## + Fold09.Rep02: gamma=0.25, lambda=0.75 
## - Fold09.Rep02: gamma=0.25, lambda=0.75 
## + Fold09.Rep02: gamma=0.50, lambda=0.75 
## - Fold09.Rep02: gamma=0.50, lambda=0.75 
## + Fold09.Rep02: gamma=0.75, lambda=0.75 
## - Fold09.Rep02: gamma=0.75, lambda=0.75 
## + Fold09.Rep02: gamma=1.00, lambda=0.75 
## - Fold09.Rep02: gamma=1.00, lambda=0.75 
## + Fold10.Rep02: gamma=0.25, lambda=0.75 
## - Fold10.Rep02: gamma=0.25, lambda=0.75 
## + Fold10.Rep02: gamma=0.50, lambda=0.75 
## - Fold10.Rep02: gamma=0.50, lambda=0.75 
## + Fold10.Rep02: gamma=0.75, lambda=0.75 
## - Fold10.Rep02: gamma=0.75, lambda=0.75 
## + Fold10.Rep02: gamma=1.00, lambda=0.75 
## - Fold10.Rep02: gamma=1.00, lambda=0.75 
## + Fold01.Rep03: gamma=0.25, lambda=0.75 
## - Fold01.Rep03: gamma=0.25, lambda=0.75 
## + Fold01.Rep03: gamma=0.50, lambda=0.75 
## - Fold01.Rep03: gamma=0.50, lambda=0.75 
## + Fold01.Rep03: gamma=0.75, lambda=0.75 
## - Fold01.Rep03: gamma=0.75, lambda=0.75 
## + Fold01.Rep03: gamma=1.00, lambda=0.75 
## - Fold01.Rep03: gamma=1.00, lambda=0.75 
## + Fold02.Rep03: gamma=0.25, lambda=0.75 
## - Fold02.Rep03: gamma=0.25, lambda=0.75 
## + Fold02.Rep03: gamma=0.50, lambda=0.75 
## - Fold02.Rep03: gamma=0.50, lambda=0.75 
## + Fold02.Rep03: gamma=0.75, lambda=0.75 
## - Fold02.Rep03: gamma=0.75, lambda=0.75 
## + Fold02.Rep03: gamma=1.00, lambda=0.75 
## - Fold02.Rep03: gamma=1.00, lambda=0.75 
## + Fold03.Rep03: gamma=0.25, lambda=0.75 
## - Fold03.Rep03: gamma=0.25, lambda=0.75 
## + Fold03.Rep03: gamma=0.50, lambda=0.75 
## - Fold03.Rep03: gamma=0.50, lambda=0.75 
## + Fold03.Rep03: gamma=0.75, lambda=0.75 
## - Fold03.Rep03: gamma=0.75, lambda=0.75 
## + Fold03.Rep03: gamma=1.00, lambda=0.75 
## - Fold03.Rep03: gamma=1.00, lambda=0.75 
## + Fold04.Rep03: gamma=0.25, lambda=0.75 
## - Fold04.Rep03: gamma=0.25, lambda=0.75 
## + Fold04.Rep03: gamma=0.50, lambda=0.75 
## - Fold04.Rep03: gamma=0.50, lambda=0.75 
## + Fold04.Rep03: gamma=0.75, lambda=0.75 
## - Fold04.Rep03: gamma=0.75, lambda=0.75 
## + Fold04.Rep03: gamma=1.00, lambda=0.75 
## - Fold04.Rep03: gamma=1.00, lambda=0.75 
## + Fold05.Rep03: gamma=0.25, lambda=0.75 
## - Fold05.Rep03: gamma=0.25, lambda=0.75 
## + Fold05.Rep03: gamma=0.50, lambda=0.75 
## - Fold05.Rep03: gamma=0.50, lambda=0.75 
## + Fold05.Rep03: gamma=0.75, lambda=0.75 
## - Fold05.Rep03: gamma=0.75, lambda=0.75 
## + Fold05.Rep03: gamma=1.00, lambda=0.75 
## - Fold05.Rep03: gamma=1.00, lambda=0.75 
## + Fold06.Rep03: gamma=0.25, lambda=0.75 
## - Fold06.Rep03: gamma=0.25, lambda=0.75 
## + Fold06.Rep03: gamma=0.50, lambda=0.75 
## - Fold06.Rep03: gamma=0.50, lambda=0.75 
## + Fold06.Rep03: gamma=0.75, lambda=0.75 
## - Fold06.Rep03: gamma=0.75, lambda=0.75 
## + Fold06.Rep03: gamma=1.00, lambda=0.75 
## - Fold06.Rep03: gamma=1.00, lambda=0.75 
## + Fold07.Rep03: gamma=0.25, lambda=0.75 
## - Fold07.Rep03: gamma=0.25, lambda=0.75 
## + Fold07.Rep03: gamma=0.50, lambda=0.75 
## - Fold07.Rep03: gamma=0.50, lambda=0.75 
## + Fold07.Rep03: gamma=0.75, lambda=0.75 
## - Fold07.Rep03: gamma=0.75, lambda=0.75 
## + Fold07.Rep03: gamma=1.00, lambda=0.75 
## - Fold07.Rep03: gamma=1.00, lambda=0.75 
## + Fold08.Rep03: gamma=0.25, lambda=0.75 
## - Fold08.Rep03: gamma=0.25, lambda=0.75 
## + Fold08.Rep03: gamma=0.50, lambda=0.75 
## - Fold08.Rep03: gamma=0.50, lambda=0.75 
## + Fold08.Rep03: gamma=0.75, lambda=0.75 
## - Fold08.Rep03: gamma=0.75, lambda=0.75 
## + Fold08.Rep03: gamma=1.00, lambda=0.75 
## - Fold08.Rep03: gamma=1.00, lambda=0.75 
## + Fold09.Rep03: gamma=0.25, lambda=0.75 
## - Fold09.Rep03: gamma=0.25, lambda=0.75 
## + Fold09.Rep03: gamma=0.50, lambda=0.75 
## - Fold09.Rep03: gamma=0.50, lambda=0.75 
## + Fold09.Rep03: gamma=0.75, lambda=0.75 
## - Fold09.Rep03: gamma=0.75, lambda=0.75 
## + Fold09.Rep03: gamma=1.00, lambda=0.75 
## - Fold09.Rep03: gamma=1.00, lambda=0.75 
## + Fold10.Rep03: gamma=0.25, lambda=0.75 
## - Fold10.Rep03: gamma=0.25, lambda=0.75 
## + Fold10.Rep03: gamma=0.50, lambda=0.75 
## - Fold10.Rep03: gamma=0.50, lambda=0.75 
## + Fold10.Rep03: gamma=0.75, lambda=0.75 
## - Fold10.Rep03: gamma=0.75, lambda=0.75 
## + Fold10.Rep03: gamma=1.00, lambda=0.75 
## - Fold10.Rep03: gamma=1.00, lambda=0.75 
## + Fold01.Rep04: gamma=0.25, lambda=0.75 
## - Fold01.Rep04: gamma=0.25, lambda=0.75 
## + Fold01.Rep04: gamma=0.50, lambda=0.75 
## - Fold01.Rep04: gamma=0.50, lambda=0.75 
## + Fold01.Rep04: gamma=0.75, lambda=0.75 
## - Fold01.Rep04: gamma=0.75, lambda=0.75 
## + Fold01.Rep04: gamma=1.00, lambda=0.75 
## - Fold01.Rep04: gamma=1.00, lambda=0.75 
## + Fold02.Rep04: gamma=0.25, lambda=0.75 
## - Fold02.Rep04: gamma=0.25, lambda=0.75 
## + Fold02.Rep04: gamma=0.50, lambda=0.75 
## - Fold02.Rep04: gamma=0.50, lambda=0.75 
## + Fold02.Rep04: gamma=0.75, lambda=0.75 
## - Fold02.Rep04: gamma=0.75, lambda=0.75 
## + Fold02.Rep04: gamma=1.00, lambda=0.75 
## - Fold02.Rep04: gamma=1.00, lambda=0.75 
## + Fold03.Rep04: gamma=0.25, lambda=0.75 
## - Fold03.Rep04: gamma=0.25, lambda=0.75 
## + Fold03.Rep04: gamma=0.50, lambda=0.75 
## - Fold03.Rep04: gamma=0.50, lambda=0.75 
## + Fold03.Rep04: gamma=0.75, lambda=0.75 
## - Fold03.Rep04: gamma=0.75, lambda=0.75 
## + Fold03.Rep04: gamma=1.00, lambda=0.75 
## - Fold03.Rep04: gamma=1.00, lambda=0.75 
## + Fold04.Rep04: gamma=0.25, lambda=0.75 
## - Fold04.Rep04: gamma=0.25, lambda=0.75 
## + Fold04.Rep04: gamma=0.50, lambda=0.75 
## - Fold04.Rep04: gamma=0.50, lambda=0.75 
## + Fold04.Rep04: gamma=0.75, lambda=0.75 
## - Fold04.Rep04: gamma=0.75, lambda=0.75 
## + Fold04.Rep04: gamma=1.00, lambda=0.75 
## - Fold04.Rep04: gamma=1.00, lambda=0.75 
## + Fold05.Rep04: gamma=0.25, lambda=0.75 
## - Fold05.Rep04: gamma=0.25, lambda=0.75 
## + Fold05.Rep04: gamma=0.50, lambda=0.75 
## - Fold05.Rep04: gamma=0.50, lambda=0.75 
## + Fold05.Rep04: gamma=0.75, lambda=0.75 
## - Fold05.Rep04: gamma=0.75, lambda=0.75 
## + Fold05.Rep04: gamma=1.00, lambda=0.75 
## - Fold05.Rep04: gamma=1.00, lambda=0.75 
## + Fold06.Rep04: gamma=0.25, lambda=0.75 
## - Fold06.Rep04: gamma=0.25, lambda=0.75 
## + Fold06.Rep04: gamma=0.50, lambda=0.75 
## - Fold06.Rep04: gamma=0.50, lambda=0.75 
## + Fold06.Rep04: gamma=0.75, lambda=0.75 
## - Fold06.Rep04: gamma=0.75, lambda=0.75 
## + Fold06.Rep04: gamma=1.00, lambda=0.75 
## - Fold06.Rep04: gamma=1.00, lambda=0.75 
## + Fold07.Rep04: gamma=0.25, lambda=0.75 
## - Fold07.Rep04: gamma=0.25, lambda=0.75 
## + Fold07.Rep04: gamma=0.50, lambda=0.75 
## - Fold07.Rep04: gamma=0.50, lambda=0.75 
## + Fold07.Rep04: gamma=0.75, lambda=0.75 
## - Fold07.Rep04: gamma=0.75, lambda=0.75 
## + Fold07.Rep04: gamma=1.00, lambda=0.75 
## - Fold07.Rep04: gamma=1.00, lambda=0.75 
## + Fold08.Rep04: gamma=0.25, lambda=0.75 
## - Fold08.Rep04: gamma=0.25, lambda=0.75 
## + Fold08.Rep04: gamma=0.50, lambda=0.75 
## - Fold08.Rep04: gamma=0.50, lambda=0.75 
## + Fold08.Rep04: gamma=0.75, lambda=0.75 
## - Fold08.Rep04: gamma=0.75, lambda=0.75 
## + Fold08.Rep04: gamma=1.00, lambda=0.75 
## - Fold08.Rep04: gamma=1.00, lambda=0.75 
## + Fold09.Rep04: gamma=0.25, lambda=0.75 
## - Fold09.Rep04: gamma=0.25, lambda=0.75 
## + Fold09.Rep04: gamma=0.50, lambda=0.75 
## - Fold09.Rep04: gamma=0.50, lambda=0.75 
## + Fold09.Rep04: gamma=0.75, lambda=0.75 
## - Fold09.Rep04: gamma=0.75, lambda=0.75 
## + Fold09.Rep04: gamma=1.00, lambda=0.75 
## - Fold09.Rep04: gamma=1.00, lambda=0.75 
## + Fold10.Rep04: gamma=0.25, lambda=0.75 
## - Fold10.Rep04: gamma=0.25, lambda=0.75 
## + Fold10.Rep04: gamma=0.50, lambda=0.75 
## - Fold10.Rep04: gamma=0.50, lambda=0.75 
## + Fold10.Rep04: gamma=0.75, lambda=0.75 
## - Fold10.Rep04: gamma=0.75, lambda=0.75 
## + Fold10.Rep04: gamma=1.00, lambda=0.75 
## - Fold10.Rep04: gamma=1.00, lambda=0.75 
## + Fold01.Rep05: gamma=0.25, lambda=0.75 
## - Fold01.Rep05: gamma=0.25, lambda=0.75 
## + Fold01.Rep05: gamma=0.50, lambda=0.75 
## - Fold01.Rep05: gamma=0.50, lambda=0.75 
## + Fold01.Rep05: gamma=0.75, lambda=0.75 
## - Fold01.Rep05: gamma=0.75, lambda=0.75 
## + Fold01.Rep05: gamma=1.00, lambda=0.75 
## - Fold01.Rep05: gamma=1.00, lambda=0.75 
## + Fold02.Rep05: gamma=0.25, lambda=0.75 
## - Fold02.Rep05: gamma=0.25, lambda=0.75 
## + Fold02.Rep05: gamma=0.50, lambda=0.75 
## - Fold02.Rep05: gamma=0.50, lambda=0.75 
## + Fold02.Rep05: gamma=0.75, lambda=0.75 
## - Fold02.Rep05: gamma=0.75, lambda=0.75 
## + Fold02.Rep05: gamma=1.00, lambda=0.75 
## - Fold02.Rep05: gamma=1.00, lambda=0.75 
## + Fold03.Rep05: gamma=0.25, lambda=0.75 
## - Fold03.Rep05: gamma=0.25, lambda=0.75 
## + Fold03.Rep05: gamma=0.50, lambda=0.75 
## - Fold03.Rep05: gamma=0.50, lambda=0.75 
## + Fold03.Rep05: gamma=0.75, lambda=0.75 
## - Fold03.Rep05: gamma=0.75, lambda=0.75 
## + Fold03.Rep05: gamma=1.00, lambda=0.75 
## - Fold03.Rep05: gamma=1.00, lambda=0.75 
## + Fold04.Rep05: gamma=0.25, lambda=0.75 
## - Fold04.Rep05: gamma=0.25, lambda=0.75 
## + Fold04.Rep05: gamma=0.50, lambda=0.75 
## - Fold04.Rep05: gamma=0.50, lambda=0.75 
## + Fold04.Rep05: gamma=0.75, lambda=0.75 
## - Fold04.Rep05: gamma=0.75, lambda=0.75 
## + Fold04.Rep05: gamma=1.00, lambda=0.75 
## - Fold04.Rep05: gamma=1.00, lambda=0.75 
## + Fold05.Rep05: gamma=0.25, lambda=0.75 
## - Fold05.Rep05: gamma=0.25, lambda=0.75 
## + Fold05.Rep05: gamma=0.50, lambda=0.75 
## - Fold05.Rep05: gamma=0.50, lambda=0.75 
## + Fold05.Rep05: gamma=0.75, lambda=0.75 
## - Fold05.Rep05: gamma=0.75, lambda=0.75 
## + Fold05.Rep05: gamma=1.00, lambda=0.75 
## - Fold05.Rep05: gamma=1.00, lambda=0.75 
## + Fold06.Rep05: gamma=0.25, lambda=0.75 
## - Fold06.Rep05: gamma=0.25, lambda=0.75 
## + Fold06.Rep05: gamma=0.50, lambda=0.75 
## - Fold06.Rep05: gamma=0.50, lambda=0.75 
## + Fold06.Rep05: gamma=0.75, lambda=0.75 
## - Fold06.Rep05: gamma=0.75, lambda=0.75 
## + Fold06.Rep05: gamma=1.00, lambda=0.75 
## - Fold06.Rep05: gamma=1.00, lambda=0.75 
## + Fold07.Rep05: gamma=0.25, lambda=0.75 
## - Fold07.Rep05: gamma=0.25, lambda=0.75 
## + Fold07.Rep05: gamma=0.50, lambda=0.75 
## - Fold07.Rep05: gamma=0.50, lambda=0.75 
## + Fold07.Rep05: gamma=0.75, lambda=0.75 
## - Fold07.Rep05: gamma=0.75, lambda=0.75 
## + Fold07.Rep05: gamma=1.00, lambda=0.75 
## - Fold07.Rep05: gamma=1.00, lambda=0.75 
## + Fold08.Rep05: gamma=0.25, lambda=0.75 
## - Fold08.Rep05: gamma=0.25, lambda=0.75 
## + Fold08.Rep05: gamma=0.50, lambda=0.75 
## - Fold08.Rep05: gamma=0.50, lambda=0.75 
## + Fold08.Rep05: gamma=0.75, lambda=0.75 
## - Fold08.Rep05: gamma=0.75, lambda=0.75 
## + Fold08.Rep05: gamma=1.00, lambda=0.75 
## - Fold08.Rep05: gamma=1.00, lambda=0.75 
## + Fold09.Rep05: gamma=0.25, lambda=0.75 
## - Fold09.Rep05: gamma=0.25, lambda=0.75 
## + Fold09.Rep05: gamma=0.50, lambda=0.75 
## - Fold09.Rep05: gamma=0.50, lambda=0.75 
## + Fold09.Rep05: gamma=0.75, lambda=0.75 
## - Fold09.Rep05: gamma=0.75, lambda=0.75 
## + Fold09.Rep05: gamma=1.00, lambda=0.75 
## - Fold09.Rep05: gamma=1.00, lambda=0.75 
## + Fold10.Rep05: gamma=0.25, lambda=0.75 
## - Fold10.Rep05: gamma=0.25, lambda=0.75 
## + Fold10.Rep05: gamma=0.50, lambda=0.75 
## - Fold10.Rep05: gamma=0.50, lambda=0.75 
## + Fold10.Rep05: gamma=0.75, lambda=0.75 
## - Fold10.Rep05: gamma=0.75, lambda=0.75 
## + Fold10.Rep05: gamma=1.00, lambda=0.75 
## - Fold10.Rep05: gamma=1.00, lambda=0.75 
## + Fold01.Rep06: gamma=0.25, lambda=0.75 
## - Fold01.Rep06: gamma=0.25, lambda=0.75 
## + Fold01.Rep06: gamma=0.50, lambda=0.75 
## - Fold01.Rep06: gamma=0.50, lambda=0.75 
## + Fold01.Rep06: gamma=0.75, lambda=0.75 
## - Fold01.Rep06: gamma=0.75, lambda=0.75 
## + Fold01.Rep06: gamma=1.00, lambda=0.75 
## - Fold01.Rep06: gamma=1.00, lambda=0.75 
## + Fold02.Rep06: gamma=0.25, lambda=0.75 
## - Fold02.Rep06: gamma=0.25, lambda=0.75 
## + Fold02.Rep06: gamma=0.50, lambda=0.75 
## - Fold02.Rep06: gamma=0.50, lambda=0.75 
## + Fold02.Rep06: gamma=0.75, lambda=0.75 
## - Fold02.Rep06: gamma=0.75, lambda=0.75 
## + Fold02.Rep06: gamma=1.00, lambda=0.75 
## - Fold02.Rep06: gamma=1.00, lambda=0.75 
## + Fold03.Rep06: gamma=0.25, lambda=0.75 
## - Fold03.Rep06: gamma=0.25, lambda=0.75 
## + Fold03.Rep06: gamma=0.50, lambda=0.75 
## - Fold03.Rep06: gamma=0.50, lambda=0.75 
## + Fold03.Rep06: gamma=0.75, lambda=0.75 
## - Fold03.Rep06: gamma=0.75, lambda=0.75 
## + Fold03.Rep06: gamma=1.00, lambda=0.75 
## - Fold03.Rep06: gamma=1.00, lambda=0.75 
## + Fold04.Rep06: gamma=0.25, lambda=0.75 
## - Fold04.Rep06: gamma=0.25, lambda=0.75 
## + Fold04.Rep06: gamma=0.50, lambda=0.75 
## - Fold04.Rep06: gamma=0.50, lambda=0.75 
## + Fold04.Rep06: gamma=0.75, lambda=0.75 
## - Fold04.Rep06: gamma=0.75, lambda=0.75 
## + Fold04.Rep06: gamma=1.00, lambda=0.75 
## - Fold04.Rep06: gamma=1.00, lambda=0.75 
## + Fold05.Rep06: gamma=0.25, lambda=0.75 
## - Fold05.Rep06: gamma=0.25, lambda=0.75 
## + Fold05.Rep06: gamma=0.50, lambda=0.75 
## - Fold05.Rep06: gamma=0.50, lambda=0.75 
## + Fold05.Rep06: gamma=0.75, lambda=0.75 
## - Fold05.Rep06: gamma=0.75, lambda=0.75 
## + Fold05.Rep06: gamma=1.00, lambda=0.75 
## - Fold05.Rep06: gamma=1.00, lambda=0.75 
## + Fold06.Rep06: gamma=0.25, lambda=0.75 
## - Fold06.Rep06: gamma=0.25, lambda=0.75 
## + Fold06.Rep06: gamma=0.50, lambda=0.75 
## - Fold06.Rep06: gamma=0.50, lambda=0.75 
## + Fold06.Rep06: gamma=0.75, lambda=0.75 
## - Fold06.Rep06: gamma=0.75, lambda=0.75 
## + Fold06.Rep06: gamma=1.00, lambda=0.75 
## - Fold06.Rep06: gamma=1.00, lambda=0.75 
## + Fold07.Rep06: gamma=0.25, lambda=0.75 
## - Fold07.Rep06: gamma=0.25, lambda=0.75 
## + Fold07.Rep06: gamma=0.50, lambda=0.75 
## - Fold07.Rep06: gamma=0.50, lambda=0.75 
## + Fold07.Rep06: gamma=0.75, lambda=0.75 
## - Fold07.Rep06: gamma=0.75, lambda=0.75 
## + Fold07.Rep06: gamma=1.00, lambda=0.75 
## - Fold07.Rep06: gamma=1.00, lambda=0.75 
## + Fold08.Rep06: gamma=0.25, lambda=0.75 
## - Fold08.Rep06: gamma=0.25, lambda=0.75 
## + Fold08.Rep06: gamma=0.50, lambda=0.75 
## - Fold08.Rep06: gamma=0.50, lambda=0.75 
## + Fold08.Rep06: gamma=0.75, lambda=0.75 
## - Fold08.Rep06: gamma=0.75, lambda=0.75 
## + Fold08.Rep06: gamma=1.00, lambda=0.75 
## - Fold08.Rep06: gamma=1.00, lambda=0.75 
## + Fold09.Rep06: gamma=0.25, lambda=0.75 
## - Fold09.Rep06: gamma=0.25, lambda=0.75 
## + Fold09.Rep06: gamma=0.50, lambda=0.75 
## - Fold09.Rep06: gamma=0.50, lambda=0.75 
## + Fold09.Rep06: gamma=0.75, lambda=0.75 
## - Fold09.Rep06: gamma=0.75, lambda=0.75 
## + Fold09.Rep06: gamma=1.00, lambda=0.75 
## - Fold09.Rep06: gamma=1.00, lambda=0.75 
## + Fold10.Rep06: gamma=0.25, lambda=0.75 
## - Fold10.Rep06: gamma=0.25, lambda=0.75 
## + Fold10.Rep06: gamma=0.50, lambda=0.75 
## - Fold10.Rep06: gamma=0.50, lambda=0.75 
## + Fold10.Rep06: gamma=0.75, lambda=0.75 
## - Fold10.Rep06: gamma=0.75, lambda=0.75 
## + Fold10.Rep06: gamma=1.00, lambda=0.75 
## - Fold10.Rep06: gamma=1.00, lambda=0.75 
## + Fold01.Rep07: gamma=0.25, lambda=0.75 
## - Fold01.Rep07: gamma=0.25, lambda=0.75 
## + Fold01.Rep07: gamma=0.50, lambda=0.75 
## - Fold01.Rep07: gamma=0.50, lambda=0.75 
## + Fold01.Rep07: gamma=0.75, lambda=0.75 
## - Fold01.Rep07: gamma=0.75, lambda=0.75 
## + Fold01.Rep07: gamma=1.00, lambda=0.75 
## - Fold01.Rep07: gamma=1.00, lambda=0.75 
## + Fold02.Rep07: gamma=0.25, lambda=0.75 
## - Fold02.Rep07: gamma=0.25, lambda=0.75 
## + Fold02.Rep07: gamma=0.50, lambda=0.75 
## - Fold02.Rep07: gamma=0.50, lambda=0.75 
## + Fold02.Rep07: gamma=0.75, lambda=0.75 
## - Fold02.Rep07: gamma=0.75, lambda=0.75 
## + Fold02.Rep07: gamma=1.00, lambda=0.75 
## - Fold02.Rep07: gamma=1.00, lambda=0.75 
## + Fold03.Rep07: gamma=0.25, lambda=0.75 
## - Fold03.Rep07: gamma=0.25, lambda=0.75 
## + Fold03.Rep07: gamma=0.50, lambda=0.75 
## - Fold03.Rep07: gamma=0.50, lambda=0.75 
## + Fold03.Rep07: gamma=0.75, lambda=0.75 
## - Fold03.Rep07: gamma=0.75, lambda=0.75 
## + Fold03.Rep07: gamma=1.00, lambda=0.75 
## - Fold03.Rep07: gamma=1.00, lambda=0.75 
## + Fold04.Rep07: gamma=0.25, lambda=0.75 
## - Fold04.Rep07: gamma=0.25, lambda=0.75 
## + Fold04.Rep07: gamma=0.50, lambda=0.75 
## - Fold04.Rep07: gamma=0.50, lambda=0.75 
## + Fold04.Rep07: gamma=0.75, lambda=0.75 
## - Fold04.Rep07: gamma=0.75, lambda=0.75 
## + Fold04.Rep07: gamma=1.00, lambda=0.75 
## - Fold04.Rep07: gamma=1.00, lambda=0.75 
## + Fold05.Rep07: gamma=0.25, lambda=0.75 
## - Fold05.Rep07: gamma=0.25, lambda=0.75 
## + Fold05.Rep07: gamma=0.50, lambda=0.75 
## - Fold05.Rep07: gamma=0.50, lambda=0.75 
## + Fold05.Rep07: gamma=0.75, lambda=0.75 
## - Fold05.Rep07: gamma=0.75, lambda=0.75 
## + Fold05.Rep07: gamma=1.00, lambda=0.75 
## - Fold05.Rep07: gamma=1.00, lambda=0.75 
## + Fold06.Rep07: gamma=0.25, lambda=0.75 
## - Fold06.Rep07: gamma=0.25, lambda=0.75 
## + Fold06.Rep07: gamma=0.50, lambda=0.75 
## - Fold06.Rep07: gamma=0.50, lambda=0.75 
## + Fold06.Rep07: gamma=0.75, lambda=0.75 
## - Fold06.Rep07: gamma=0.75, lambda=0.75 
## + Fold06.Rep07: gamma=1.00, lambda=0.75 
## - Fold06.Rep07: gamma=1.00, lambda=0.75 
## + Fold07.Rep07: gamma=0.25, lambda=0.75 
## - Fold07.Rep07: gamma=0.25, lambda=0.75 
## + Fold07.Rep07: gamma=0.50, lambda=0.75 
## - Fold07.Rep07: gamma=0.50, lambda=0.75 
## + Fold07.Rep07: gamma=0.75, lambda=0.75 
## - Fold07.Rep07: gamma=0.75, lambda=0.75 
## + Fold07.Rep07: gamma=1.00, lambda=0.75 
## - Fold07.Rep07: gamma=1.00, lambda=0.75 
## + Fold08.Rep07: gamma=0.25, lambda=0.75 
## - Fold08.Rep07: gamma=0.25, lambda=0.75 
## + Fold08.Rep07: gamma=0.50, lambda=0.75 
## - Fold08.Rep07: gamma=0.50, lambda=0.75 
## + Fold08.Rep07: gamma=0.75, lambda=0.75 
## - Fold08.Rep07: gamma=0.75, lambda=0.75 
## + Fold08.Rep07: gamma=1.00, lambda=0.75 
## - Fold08.Rep07: gamma=1.00, lambda=0.75 
## + Fold09.Rep07: gamma=0.25, lambda=0.75 
## - Fold09.Rep07: gamma=0.25, lambda=0.75 
## + Fold09.Rep07: gamma=0.50, lambda=0.75 
## - Fold09.Rep07: gamma=0.50, lambda=0.75 
## + Fold09.Rep07: gamma=0.75, lambda=0.75 
## - Fold09.Rep07: gamma=0.75, lambda=0.75 
## + Fold09.Rep07: gamma=1.00, lambda=0.75 
## - Fold09.Rep07: gamma=1.00, lambda=0.75 
## + Fold10.Rep07: gamma=0.25, lambda=0.75 
## - Fold10.Rep07: gamma=0.25, lambda=0.75 
## + Fold10.Rep07: gamma=0.50, lambda=0.75 
## - Fold10.Rep07: gamma=0.50, lambda=0.75 
## + Fold10.Rep07: gamma=0.75, lambda=0.75 
## - Fold10.Rep07: gamma=0.75, lambda=0.75 
## + Fold10.Rep07: gamma=1.00, lambda=0.75 
## - Fold10.Rep07: gamma=1.00, lambda=0.75 
## + Fold01.Rep08: gamma=0.25, lambda=0.75 
## - Fold01.Rep08: gamma=0.25, lambda=0.75 
## + Fold01.Rep08: gamma=0.50, lambda=0.75 
## - Fold01.Rep08: gamma=0.50, lambda=0.75 
## + Fold01.Rep08: gamma=0.75, lambda=0.75 
## - Fold01.Rep08: gamma=0.75, lambda=0.75 
## + Fold01.Rep08: gamma=1.00, lambda=0.75 
## - Fold01.Rep08: gamma=1.00, lambda=0.75 
## + Fold02.Rep08: gamma=0.25, lambda=0.75 
## - Fold02.Rep08: gamma=0.25, lambda=0.75 
## + Fold02.Rep08: gamma=0.50, lambda=0.75 
## - Fold02.Rep08: gamma=0.50, lambda=0.75 
## + Fold02.Rep08: gamma=0.75, lambda=0.75 
## - Fold02.Rep08: gamma=0.75, lambda=0.75 
## + Fold02.Rep08: gamma=1.00, lambda=0.75 
## - Fold02.Rep08: gamma=1.00, lambda=0.75 
## + Fold03.Rep08: gamma=0.25, lambda=0.75 
## - Fold03.Rep08: gamma=0.25, lambda=0.75 
## + Fold03.Rep08: gamma=0.50, lambda=0.75 
## - Fold03.Rep08: gamma=0.50, lambda=0.75 
## + Fold03.Rep08: gamma=0.75, lambda=0.75 
## - Fold03.Rep08: gamma=0.75, lambda=0.75 
## + Fold03.Rep08: gamma=1.00, lambda=0.75 
## - Fold03.Rep08: gamma=1.00, lambda=0.75 
## + Fold04.Rep08: gamma=0.25, lambda=0.75 
## - Fold04.Rep08: gamma=0.25, lambda=0.75 
## + Fold04.Rep08: gamma=0.50, lambda=0.75 
## - Fold04.Rep08: gamma=0.50, lambda=0.75 
## + Fold04.Rep08: gamma=0.75, lambda=0.75 
## - Fold04.Rep08: gamma=0.75, lambda=0.75 
## + Fold04.Rep08: gamma=1.00, lambda=0.75 
## - Fold04.Rep08: gamma=1.00, lambda=0.75 
## + Fold05.Rep08: gamma=0.25, lambda=0.75 
## - Fold05.Rep08: gamma=0.25, lambda=0.75 
## + Fold05.Rep08: gamma=0.50, lambda=0.75 
## - Fold05.Rep08: gamma=0.50, lambda=0.75 
## + Fold05.Rep08: gamma=0.75, lambda=0.75 
## - Fold05.Rep08: gamma=0.75, lambda=0.75 
## + Fold05.Rep08: gamma=1.00, lambda=0.75 
## - Fold05.Rep08: gamma=1.00, lambda=0.75 
## + Fold06.Rep08: gamma=0.25, lambda=0.75 
## - Fold06.Rep08: gamma=0.25, lambda=0.75 
## + Fold06.Rep08: gamma=0.50, lambda=0.75 
## - Fold06.Rep08: gamma=0.50, lambda=0.75 
## + Fold06.Rep08: gamma=0.75, lambda=0.75 
## - Fold06.Rep08: gamma=0.75, lambda=0.75 
## + Fold06.Rep08: gamma=1.00, lambda=0.75 
## - Fold06.Rep08: gamma=1.00, lambda=0.75 
## + Fold07.Rep08: gamma=0.25, lambda=0.75 
## - Fold07.Rep08: gamma=0.25, lambda=0.75 
## + Fold07.Rep08: gamma=0.50, lambda=0.75 
## - Fold07.Rep08: gamma=0.50, lambda=0.75 
## + Fold07.Rep08: gamma=0.75, lambda=0.75 
## - Fold07.Rep08: gamma=0.75, lambda=0.75 
## + Fold07.Rep08: gamma=1.00, lambda=0.75 
## - Fold07.Rep08: gamma=1.00, lambda=0.75 
## + Fold08.Rep08: gamma=0.25, lambda=0.75 
## - Fold08.Rep08: gamma=0.25, lambda=0.75 
## + Fold08.Rep08: gamma=0.50, lambda=0.75 
## - Fold08.Rep08: gamma=0.50, lambda=0.75 
## + Fold08.Rep08: gamma=0.75, lambda=0.75 
## - Fold08.Rep08: gamma=0.75, lambda=0.75 
## + Fold08.Rep08: gamma=1.00, lambda=0.75 
## - Fold08.Rep08: gamma=1.00, lambda=0.75 
## + Fold09.Rep08: gamma=0.25, lambda=0.75 
## - Fold09.Rep08: gamma=0.25, lambda=0.75 
## + Fold09.Rep08: gamma=0.50, lambda=0.75 
## - Fold09.Rep08: gamma=0.50, lambda=0.75 
## + Fold09.Rep08: gamma=0.75, lambda=0.75 
## - Fold09.Rep08: gamma=0.75, lambda=0.75 
## + Fold09.Rep08: gamma=1.00, lambda=0.75 
## - Fold09.Rep08: gamma=1.00, lambda=0.75 
## + Fold10.Rep08: gamma=0.25, lambda=0.75 
## - Fold10.Rep08: gamma=0.25, lambda=0.75 
## + Fold10.Rep08: gamma=0.50, lambda=0.75 
## - Fold10.Rep08: gamma=0.50, lambda=0.75 
## + Fold10.Rep08: gamma=0.75, lambda=0.75 
## - Fold10.Rep08: gamma=0.75, lambda=0.75 
## + Fold10.Rep08: gamma=1.00, lambda=0.75 
## - Fold10.Rep08: gamma=1.00, lambda=0.75 
## + Fold01.Rep09: gamma=0.25, lambda=0.75 
## - Fold01.Rep09: gamma=0.25, lambda=0.75 
## + Fold01.Rep09: gamma=0.50, lambda=0.75 
## - Fold01.Rep09: gamma=0.50, lambda=0.75 
## + Fold01.Rep09: gamma=0.75, lambda=0.75 
## - Fold01.Rep09: gamma=0.75, lambda=0.75 
## + Fold01.Rep09: gamma=1.00, lambda=0.75 
## - Fold01.Rep09: gamma=1.00, lambda=0.75 
## + Fold02.Rep09: gamma=0.25, lambda=0.75 
## - Fold02.Rep09: gamma=0.25, lambda=0.75 
## + Fold02.Rep09: gamma=0.50, lambda=0.75 
## - Fold02.Rep09: gamma=0.50, lambda=0.75 
## + Fold02.Rep09: gamma=0.75, lambda=0.75 
## - Fold02.Rep09: gamma=0.75, lambda=0.75 
## + Fold02.Rep09: gamma=1.00, lambda=0.75 
## - Fold02.Rep09: gamma=1.00, lambda=0.75 
## + Fold03.Rep09: gamma=0.25, lambda=0.75 
## - Fold03.Rep09: gamma=0.25, lambda=0.75 
## + Fold03.Rep09: gamma=0.50, lambda=0.75 
## - Fold03.Rep09: gamma=0.50, lambda=0.75 
## + Fold03.Rep09: gamma=0.75, lambda=0.75 
## - Fold03.Rep09: gamma=0.75, lambda=0.75 
## + Fold03.Rep09: gamma=1.00, lambda=0.75 
## - Fold03.Rep09: gamma=1.00, lambda=0.75 
## + Fold04.Rep09: gamma=0.25, lambda=0.75 
## - Fold04.Rep09: gamma=0.25, lambda=0.75 
## + Fold04.Rep09: gamma=0.50, lambda=0.75 
## - Fold04.Rep09: gamma=0.50, lambda=0.75 
## + Fold04.Rep09: gamma=0.75, lambda=0.75 
## - Fold04.Rep09: gamma=0.75, lambda=0.75 
## + Fold04.Rep09: gamma=1.00, lambda=0.75 
## - Fold04.Rep09: gamma=1.00, lambda=0.75 
## + Fold05.Rep09: gamma=0.25, lambda=0.75 
## - Fold05.Rep09: gamma=0.25, lambda=0.75 
## + Fold05.Rep09: gamma=0.50, lambda=0.75 
## - Fold05.Rep09: gamma=0.50, lambda=0.75 
## + Fold05.Rep09: gamma=0.75, lambda=0.75 
## - Fold05.Rep09: gamma=0.75, lambda=0.75 
## + Fold05.Rep09: gamma=1.00, lambda=0.75 
## - Fold05.Rep09: gamma=1.00, lambda=0.75 
## + Fold06.Rep09: gamma=0.25, lambda=0.75 
## - Fold06.Rep09: gamma=0.25, lambda=0.75 
## + Fold06.Rep09: gamma=0.50, lambda=0.75 
## - Fold06.Rep09: gamma=0.50, lambda=0.75 
## + Fold06.Rep09: gamma=0.75, lambda=0.75 
## - Fold06.Rep09: gamma=0.75, lambda=0.75 
## + Fold06.Rep09: gamma=1.00, lambda=0.75 
## - Fold06.Rep09: gamma=1.00, lambda=0.75 
## + Fold07.Rep09: gamma=0.25, lambda=0.75 
## - Fold07.Rep09: gamma=0.25, lambda=0.75 
## + Fold07.Rep09: gamma=0.50, lambda=0.75 
## - Fold07.Rep09: gamma=0.50, lambda=0.75 
## + Fold07.Rep09: gamma=0.75, lambda=0.75 
## - Fold07.Rep09: gamma=0.75, lambda=0.75 
## + Fold07.Rep09: gamma=1.00, lambda=0.75 
## - Fold07.Rep09: gamma=1.00, lambda=0.75 
## + Fold08.Rep09: gamma=0.25, lambda=0.75 
## - Fold08.Rep09: gamma=0.25, lambda=0.75 
## + Fold08.Rep09: gamma=0.50, lambda=0.75 
## - Fold08.Rep09: gamma=0.50, lambda=0.75 
## + Fold08.Rep09: gamma=0.75, lambda=0.75 
## - Fold08.Rep09: gamma=0.75, lambda=0.75 
## + Fold08.Rep09: gamma=1.00, lambda=0.75 
## - Fold08.Rep09: gamma=1.00, lambda=0.75 
## + Fold09.Rep09: gamma=0.25, lambda=0.75 
## - Fold09.Rep09: gamma=0.25, lambda=0.75 
## + Fold09.Rep09: gamma=0.50, lambda=0.75 
## - Fold09.Rep09: gamma=0.50, lambda=0.75 
## + Fold09.Rep09: gamma=0.75, lambda=0.75 
## - Fold09.Rep09: gamma=0.75, lambda=0.75 
## + Fold09.Rep09: gamma=1.00, lambda=0.75 
## - Fold09.Rep09: gamma=1.00, lambda=0.75 
## + Fold10.Rep09: gamma=0.25, lambda=0.75 
## - Fold10.Rep09: gamma=0.25, lambda=0.75 
## + Fold10.Rep09: gamma=0.50, lambda=0.75 
## - Fold10.Rep09: gamma=0.50, lambda=0.75 
## + Fold10.Rep09: gamma=0.75, lambda=0.75 
## - Fold10.Rep09: gamma=0.75, lambda=0.75 
## + Fold10.Rep09: gamma=1.00, lambda=0.75 
## - Fold10.Rep09: gamma=1.00, lambda=0.75 
## + Fold01.Rep10: gamma=0.25, lambda=0.75 
## - Fold01.Rep10: gamma=0.25, lambda=0.75 
## + Fold01.Rep10: gamma=0.50, lambda=0.75 
## - Fold01.Rep10: gamma=0.50, lambda=0.75 
## + Fold01.Rep10: gamma=0.75, lambda=0.75 
## - Fold01.Rep10: gamma=0.75, lambda=0.75 
## + Fold01.Rep10: gamma=1.00, lambda=0.75 
## - Fold01.Rep10: gamma=1.00, lambda=0.75 
## + Fold02.Rep10: gamma=0.25, lambda=0.75 
## - Fold02.Rep10: gamma=0.25, lambda=0.75 
## + Fold02.Rep10: gamma=0.50, lambda=0.75 
## - Fold02.Rep10: gamma=0.50, lambda=0.75 
## + Fold02.Rep10: gamma=0.75, lambda=0.75 
## - Fold02.Rep10: gamma=0.75, lambda=0.75 
## + Fold02.Rep10: gamma=1.00, lambda=0.75 
## - Fold02.Rep10: gamma=1.00, lambda=0.75 
## + Fold03.Rep10: gamma=0.25, lambda=0.75 
## - Fold03.Rep10: gamma=0.25, lambda=0.75 
## + Fold03.Rep10: gamma=0.50, lambda=0.75 
## - Fold03.Rep10: gamma=0.50, lambda=0.75 
## + Fold03.Rep10: gamma=0.75, lambda=0.75 
## - Fold03.Rep10: gamma=0.75, lambda=0.75 
## + Fold03.Rep10: gamma=1.00, lambda=0.75 
## - Fold03.Rep10: gamma=1.00, lambda=0.75 
## + Fold04.Rep10: gamma=0.25, lambda=0.75 
## - Fold04.Rep10: gamma=0.25, lambda=0.75 
## + Fold04.Rep10: gamma=0.50, lambda=0.75 
## - Fold04.Rep10: gamma=0.50, lambda=0.75 
## + Fold04.Rep10: gamma=0.75, lambda=0.75 
## - Fold04.Rep10: gamma=0.75, lambda=0.75 
## + Fold04.Rep10: gamma=1.00, lambda=0.75 
## - Fold04.Rep10: gamma=1.00, lambda=0.75 
## + Fold05.Rep10: gamma=0.25, lambda=0.75 
## - Fold05.Rep10: gamma=0.25, lambda=0.75 
## + Fold05.Rep10: gamma=0.50, lambda=0.75 
## - Fold05.Rep10: gamma=0.50, lambda=0.75 
## + Fold05.Rep10: gamma=0.75, lambda=0.75 
## - Fold05.Rep10: gamma=0.75, lambda=0.75 
## + Fold05.Rep10: gamma=1.00, lambda=0.75 
## - Fold05.Rep10: gamma=1.00, lambda=0.75 
## + Fold06.Rep10: gamma=0.25, lambda=0.75 
## - Fold06.Rep10: gamma=0.25, lambda=0.75 
## + Fold06.Rep10: gamma=0.50, lambda=0.75 
## - Fold06.Rep10: gamma=0.50, lambda=0.75 
## + Fold06.Rep10: gamma=0.75, lambda=0.75 
## - Fold06.Rep10: gamma=0.75, lambda=0.75 
## + Fold06.Rep10: gamma=1.00, lambda=0.75 
## - Fold06.Rep10: gamma=1.00, lambda=0.75 
## + Fold07.Rep10: gamma=0.25, lambda=0.75 
## - Fold07.Rep10: gamma=0.25, lambda=0.75 
## + Fold07.Rep10: gamma=0.50, lambda=0.75 
## - Fold07.Rep10: gamma=0.50, lambda=0.75 
## + Fold07.Rep10: gamma=0.75, lambda=0.75 
## - Fold07.Rep10: gamma=0.75, lambda=0.75 
## + Fold07.Rep10: gamma=1.00, lambda=0.75 
## - Fold07.Rep10: gamma=1.00, lambda=0.75 
## + Fold08.Rep10: gamma=0.25, lambda=0.75 
## - Fold08.Rep10: gamma=0.25, lambda=0.75 
## + Fold08.Rep10: gamma=0.50, lambda=0.75 
## - Fold08.Rep10: gamma=0.50, lambda=0.75 
## + Fold08.Rep10: gamma=0.75, lambda=0.75 
## - Fold08.Rep10: gamma=0.75, lambda=0.75 
## + Fold08.Rep10: gamma=1.00, lambda=0.75 
## - Fold08.Rep10: gamma=1.00, lambda=0.75 
## + Fold09.Rep10: gamma=0.25, lambda=0.75 
## - Fold09.Rep10: gamma=0.25, lambda=0.75 
## + Fold09.Rep10: gamma=0.50, lambda=0.75 
## - Fold09.Rep10: gamma=0.50, lambda=0.75 
## + Fold09.Rep10: gamma=0.75, lambda=0.75 
## - Fold09.Rep10: gamma=0.75, lambda=0.75 
## + Fold09.Rep10: gamma=1.00, lambda=0.75 
## - Fold09.Rep10: gamma=1.00, lambda=0.75 
## + Fold10.Rep10: gamma=0.25, lambda=0.75 
## - Fold10.Rep10: gamma=0.25, lambda=0.75 
## + Fold10.Rep10: gamma=0.50, lambda=0.75 
## - Fold10.Rep10: gamma=0.50, lambda=0.75 
## + Fold10.Rep10: gamma=0.75, lambda=0.75 
## - Fold10.Rep10: gamma=0.75, lambda=0.75 
## + Fold10.Rep10: gamma=1.00, lambda=0.75 
## - Fold10.Rep10: gamma=1.00, lambda=0.75 
## Aggregating results
## Selecting tuning parameters
## Fitting gamma = 0.25, lambda = 0.75 on full training set
plot(rdafit)

rdafit
## Regularized Discriminant Analysis 
## 
## 7720 samples
##   55 predictor
##   23 classes: 'ACC', 'BLCA', 'BRCA', 'CESC', 'COLORECTAL', 'DLBC', 'GLIOMA', 'HNSC', 'KIDNEY', 'LUNG', 'MESO', 'OV', 'PARALIVER', 'PCPG', 'PRAD', 'SARC', 'SKCM', 'STOPH', 'TGCT', 'THCA', 'THYM', 'UTERINE', 'UVM' 
## 
## No pre-processing
## Resampling: Cross-Validated (10 fold, repeated 10 times) 
## Summary of sample sizes: 6949, 6948, 6944, 6950, 6948, 6947, ... 
## Resampling results across tuning parameters:
## 
##   gamma  logLoss    AUC        prAUC      Accuracy   Kappa      Mean_F1  
##   0.25   0.3736955  0.9971095  0.7359120  0.9409595  0.9358314  0.9280266
##   0.50   0.4191939  0.9966447  0.7683977  0.9321521  0.9262610  0.9209384
##   0.75   0.5667507  0.9955382  0.7724702  0.9124616  0.9048614  0.9025944
##   1.00   1.4541206  0.9865064  0.7275851  0.8686270  0.8574948  0.8583599
##   Mean_Sensitivity  Mean_Specificity  Mean_Pos_Pred_Value  Mean_Neg_Pred_Value
##   0.9334763         0.9973037         0.9296072            0.9972794          
##   0.9261794         0.9968942         0.9236205            0.9968698          
##   0.9079277         0.9959817         0.9083787            0.9959613          
##   0.8708014         0.9939828         0.8593739            0.9938756          
##   Mean_Precision  Mean_Recall  Mean_Detection_Rate  Mean_Balanced_Accuracy
##   0.9296072       0.9334763    0.04091128           0.9653900             
##   0.9236205       0.9261794    0.04052835           0.9615368             
##   0.9083787       0.9079277    0.03967224           0.9519547             
##   0.8593739       0.8708014    0.03776639           0.9323921             
## 
## Tuning parameter 'lambda' was held constant at a value of 0.75
## Accuracy was used to select the optimal model using the largest value.
## The final values used for the model were gamma = 0.25 and lambda = 0.75.
qual_col_pals = brewer.pal.info[brewer.pal.info$category == 'qual',]
col_vector = unlist(mapply(brewer.pal, qual_col_pals$maxcolors, rownames(qual_col_pals)))
pp2 <- preProcess(gt_all2[,-1], method = c("center", "nzv", "scale", "YeoJohnson"))
moddat <- predict(pp2, gt_all2)
pal = colorRampPalette(c("red","red","red", "black", "green","green","green"))(n = 128)
dd = dist(as.matrix(moddat[,-1]), method="maximum")
hh = hclust(dd, method="ward.D2")
moddat = moddat[order(as.numeric(moddat$mod_id)),]

# --- visualisation: heatmap ---#
pdf ('../../plots/heatmap_types.pdf', width=10, height=14)
heatmap.2(as.matrix(moddat[,-1]), trace="none", labRow = NA,
          col=pal, #Rowv = NA,
          RowSideColors = col_vector[as.numeric(factor(moddat$mod_id))],
          hclustfun = function (x) {
              hclust(dist(
                  x, method="maximum")
              , method = "ward.D2")
          }, key = F
)
dev.off()
## quartz_off_screen 
##                 2
# --- visualization: hierarchial clustering dendrogram ---#
set.seed(1234)
ss = sample(1:nrow(moddat), size=1000)
ss = moddat[ss,]
rownames(ss) = paste0(1:nrow(ss), ss$mod_id)
dd = dist(as.matrix(ss[,-1]), method="maximum")
hh = hclust(dd, method="ward.D2")

ColorDendrogram(hc=hh, y=col_vector[as.numeric(factor(ss$mod_id))],
                branchlength = 6, main = "canberra"
                #labels = ss$mod_id
                )